Tag Archives: java

使用NIO进行快速的文件拷贝

  1. public static void fileCopy( File in, File out )
  2.             throws IOException
  3.     {
  4.         FileChannel inChannel = new FileInputStream( in ).getChannel();
  5.         FileChannel outChannel = new FileOutputStream( out ).getChannel();
  6.         try
  7.         {
  8. //          inChannel.transferTo(0, inChannel.size(), outChannel);      // original — apparently has trouble copying large files on Windows  
  9.             // magic number for Windows, 64Mb – 32Kb)  
  10.             int maxCount = (64 * 1024 * 1024) – (32 * 1024);
  11.             long size = inChannel.size();
  12.             long position = 0;
  13.             while ( position < size )
  14.             {
  15.                position += inChannel.transferTo( position, maxCount, outChannel );
  16.             }
  17.         }
  18.         finally
  19.         {
  20.             if ( inChannel != null )
  21.             {
  22.                inChannel.close();
  23.             }
  24.             if ( outChannel != null )
  25.             {
  26.                 outChannel.close();
  27.             }
  28.         }
  29.     }

列出文件和目录

  1. File dir = new File(“directoryName”);
  2.   String[] children = dir.list();
  3.   if (children == null) {
  4.       // Either dir does not exist or is not a directory  
  5.   } else {
  6.       for (int i=0; i < children.length; i++) {
  7.           // Get filename of file or directory  
  8.           String filename = children[i];
  9.       }
  10.   }
  11.   // It is also possible to filter the list of returned files.  
  12.   // This example does not return any files that start with `.’.  
  13.   FilenameFilter filter = new FilenameFilter() {
  14.       public boolean accept(File dir, String name) {
  15.           return !name.startsWith(“.”);
  16.       }
  17.   };
  18.   children = dir.list(filter);
  19.   // The list of files can also be retrieved as File objects  
  20.   File[] files = dir.listFiles();
  21.   // This filter only returns directories  
  22.   FileFilter fileFilter = new FileFilter() {
  23.       public boolean accept(File file) {
  24.           return file.isDirectory();
  25.       }
  26.   };
  27.   files = dir.listFiles(fileFilter);

创建ZIP和JAR文件 create zip jar

  1. import java.util.zip.*;
  2. import java.io.*;
  3. public class ZipIt {
  4.     public static void main(String args[]) throws IOException {
  5.         if (args.length < 2) {
  6.             System.err.println(“usage: java ZipIt Zip.zip file1 file2 file3”);
  7.             System.exit(-1);
  8.         }
  9.         File zipFile = new File(args[0]);
  10.         if (zipFile.exists()) {
  11.             System.err.println(“Zip file already exists, please try another”);
  12.             System.exit(-2);
  13.         }
  14.         FileOutputStream fos = new FileOutputStream(zipFile);
  15.         ZipOutputStream zos = new ZipOutputStream(fos);
  16.         int bytesRead;
  17.         byte[] buffer = new byte[1024];
  18.         CRC32 crc = new CRC32();
  19.         for (int i=1, n=args.length; i < n; i++) {
  20.             String name = args[i];
  21.             File file = new File(name);
  22.             if (!file.exists()) {
  23.                 System.err.println(“Skipping: “ + name);
  24.                 continue;
  25.             }
  26.             BufferedInputStream bis = new BufferedInputStream(
  27.                 new FileInputStream(file));
  28.             crc.reset();
  29.             while ((bytesRead = bis.read(buffer)) != –1) {
  30.                 crc.update(buffer, 0, bytesRead);
  31.             }
  32.             bis.close();
  33.             // Reset to beginning of input stream  
  34.             bis = new BufferedInputStream(
  35.                 new FileInputStream(file));
  36.             ZipEntry entry = new ZipEntry(name);
  37.             entry.setMethod(ZipEntry.STORED);
  38.             entry.setCompressedSize(file.length());
  39.             entry.setSize(file.length());
  40.             entry.setCrc(crc.getValue());
  41.             zos.putNextEntry(entry);
  42.             while ((bytesRead = bis.read(buffer)) != –1) {
  43.                 zos.write(buffer, 0, bytesRead);
  44.             }
  45.             bis.close();
  46.         }
  47.         zos.close();
  48.     }
  49. }

解析/读取XML 文件

  1. <?xml version=“1.0”?>
  2. <students>
  3.     <student>
  4.         <name>John</name>
  5.         <grade>B</grade>
  6.         <age>12</age>
  7.     </student>
  8.     <student>
  9.         <name>Mary</name>
  10.         <grade>A</grade>
  11.         <age>11</age>
  12.     </student>
  13.     <student>
  14.         <name>Simon</name>
  15.         <grade>A</grade>
  16.         <age>18</age>
  17.     </student>
  18. </students>
  1. package net.viralpatel.java.xmlparser;
  2. import java.io.File;
  3. import javax.xml.parsers.DocumentBuilder;
  4. import javax.xml.parsers.DocumentBuilderFactory;
  5. import org.w3c.dom.Document;
  6. import org.w3c.dom.Element;
  7. import org.w3c.dom.Node;
  8. import org.w3c.dom.NodeList;
  9. public class XMLParser {
  10.     public void getAllUserNames(String fileName) {
  11.         try {
  12.             DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
  13.             DocumentBuilder db = dbf.newDocumentBuilder();
  14.             File file = new File(fileName);
  15.             if (file.exists()) {
  16.                 Document doc = db.parse(file);
  17.                 Element docEle = doc.getDocumentElement();
  18.                 // Print root element of the document  
  19.                 System.out.println(“Root element of the document: “
  20.                         + docEle.getNodeName());
  21.                 NodeList studentList = docEle.getElementsByTagName(“student”);
  22.                 // Print total student elements in document  
  23.                 System.out
  24.                         .println(“Total students: “ + studentList.getLength());
  25.                 if (studentList != null && studentList.getLength() > 0) {
  26.                     for (int i = 0; i < studentList.getLength(); i++) {
  27.                         Node node = studentList.item(i);
  28.                         if (node.getNodeType() == Node.ELEMENT_NODE) {
  29.                             System.out
  30.                                     .println(“=====================”);
  31.                             Element e = (Element) node;
  32.                             NodeList nodeList = e.getElementsByTagName(“name”);
  33.                             System.out.println(“Name: “
  34.                                     + nodeList.item(0).getChildNodes().item(0)
  35.                                             .getNodeValue());
  36.                             nodeList = e.getElementsByTagName(“grade”);
  37.                             System.out.println(“Grade: “
  38.                                     + nodeList.item(0).getChildNodes().item(0)
  39.                                             .getNodeValue());
  40.                             nodeList = e.getElementsByTagName(“age”);
  41.                             System.out.println(“Age: “
  42.                                     + nodeList.item(0).getChildNodes().item(0)
  43.                                             .getNodeValue());
  44.                         }
  45.                     }
  46.                 } else {
  47.                     System.exit(1);
  48.                 }
  49.             }
  50.         } catch (Exception e) {
  51.             System.out.println(e);
  52.         }
  53.     }
  54.     public static void main(String[] args) {
  55.         XMLParser parser = new XMLParser();
  56.         parser.getAllUserNames(“c:\\test.xml”);
  57.     }
  58. }

使用iText JAR生成PDF

  1. import java.io.File;
  2. import java.io.FileOutputStream;
  3. import java.io.OutputStream;
  4. import java.util.Date;
  5. import com.lowagie.text.Document;
  6. import com.lowagie.text.Paragraph;
  7. import com.lowagie.text.pdf.PdfWriter;
  8. public class GeneratePDF {
  9.     public static void main(String[] args) {
  10.         try {
  11.             OutputStream file = new FileOutputStream(new File(“C:\\Test.pdf”));
  12.             Document document = new Document();
  13.             PdfWriter.getInstance(document, file);
  14.             document.open();
  15.             document.add(new Paragraph(“Hello Kiran”));
  16.             document.add(new Paragraph(new Date().toString()));
  17.             document.close();
  18.             file.close();
  19.         } catch (Exception e) {
  20.             e.printStackTrace();
  21.         }
  22.     }
  23. }

创建图片的缩略图 createThumbnail

  1. private void createThumbnail(String filename, int thumbWidth, int thumbHeight, int quality, String outFilename)
  2.         throws InterruptedException, FileNotFoundException, IOException
  3.     {
  4.         // load image from filename  
  5.         Image image = Toolkit.getDefaultToolkit().getImage(filename);
  6.         MediaTracker mediaTracker = new MediaTracker(new Container());
  7.         mediaTracker.addImage(image, 0);
  8.         mediaTracker.waitForID(0);
  9.         // use this to test for errors at this point: System.out.println(mediaTracker.isErrorAny());  
  10.         // determine thumbnail size from WIDTH and HEIGHT  
  11.         double thumbRatio = (double)thumbWidth / (double)thumbHeight;
  12.         int imageWidth = image.getWidth(null);
  13.         int imageHeight = image.getHeight(null);
  14.         double imageRatio = (double)imageWidth / (double)imageHeight;
  15.         if (thumbRatio < imageRatio) {
  16.             thumbHeight = (int)(thumbWidth / imageRatio);
  17.         } else {
  18.             thumbWidth = (int)(thumbHeight * imageRatio);
  19.         }
  20.         // draw original image to thumbnail image object and  
  21.         // scale it to the new size on-the-fly  
  22.         BufferedImage thumbImage = new BufferedImage(thumbWidth, thumbHeight, BufferedImage.TYPE_INT_RGB);
  23.         Graphics2D graphics2D = thumbImage.createGraphics();
  24.         graphics2D.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
  25.         graphics2D.drawImage(image, 00, thumbWidth, thumbHeight, null);
  26.         // save thumbnail image to outFilename  
  27.         BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outFilename));
  28.         JPEGImageEncoder encoder = JPEGCodec.createJPEGEncoder(out);
  29.         JPEGEncodeParam param = encoder.getDefaultJPEGEncodeParam(thumbImage);
  30.         quality = Math.max(0, Math.min(quality, 100));
  31.         param.setQuality((float)quality / 100.0f, false);
  32.         encoder.setJPEGEncodeParam(param);
  33.         encoder.encode(thumbImage);
  34.         out.close();
  35.     }

Java Reflection, 1000x Faster

A few weeks ago I got to make some of my code 1000 times faster, without changing the underlying complexity! As the title implies, this involved making Java reflection calls faster.

Let me explain my use case as well, because it’s relatively general, and a good example of why one would use reflection in the first place.

I had an interface (representing a tree node) and a slew of classes (100+) implementing this interface. The trick is that the tree is heterogeneous, each node kind can have different numbers of children, or store them differently.

I needed some code te be able to walk over such a composite tree. The simple approach is to simply add a children() method to the interface and implement it in every kind of node. Very tedious, and boilerplaty as hell.

Instead, I noted that all children were either direct fields, or aggregated in fields holding a collection of nodes. I could write a small piece of code that, with reflection, would work for every node kind!

I’ve put up a much simplified version of the code on Github. I will link the relevant parts as we go.

Initial Code

Here is the version I came up with: WalkerDemoSlowest.java

It’s fairly straightforward: get the methods of the node’s class, filter out those that are not getters, then consider only that return either a node or a collection of node. For those, invoke the method, and recursively invoke walk on the children.

Will anyone be surprised if I tell them it’s very slow?

Caching

There is a simple tweak we can apply that makes it much faster however: we can cache the methods lookup.

Here is the caching version: WalkerDemoSlow.java

It’s really the same except that for each class implementing Node, we create aClassData object that caches all the relevant getters, so we only have to look them up once. This produces a satisfying ~10x speedup.

LambdaMetafactory Magic

Unfortunately, this was still way too slow. So I took to Google, which turned out this helpful StackOverflow question.

The accepted answers proposes the use of LambdaMetafactory, a standard library class that supports lambda invocations in the language.

The details are somewhat hazy to me, but it seems that by using these facilities we can “summon the compiler” on our code and optimize the reflective access into a native invocation. That’s the working hypothesis anyhow.

Here is the code: WalkerDemoFast.java

Now, in my code, this worked wonders, unlocking another 100x speedup. While writing this article however, I wanted to demonstrate the effect with some code snippet, but didn’t manage to. I tried to give the interface three sub-classes, and to give them bogus methods to be filtered out, to no avail. The second and third version of the code would run at about the same speed.

I re-checked the original code — all seemed good. In my original code, the trees are Abstract Syntax Trees (AST) derived by parsing some source files. After fooling around some more, I noticed different results if I limited the input to the first 14 source files.

These files are relatively short (few 10s of lines) and syntactically simple. With only those, the second and third version would run at about the same speed. But add in the 15th file (a few 100s of lines) and the second version would take a whopping 36 seconds while the third version would still complete in 0.2 seconds, a ~700x difference.

My (somewhat shaky) hypothesis is that if the scenario is simple enough, the optimizer notices what you are doing and optmizes away. In more complex cases, it exhausts its optimization budget and falls back on the unoptimized version and its abysmal performance. But the optimizer is devious enough that crafting a toy example that would defeat it seems to be quite the feat.

LambdaMetafactory Possibilities

I’m somewhat intrigued about what is possible with LambdaMetafactory. In my use case, it works wonders because reflection calls are much more expensive than a simple cache lookup. But could it be used to optmize regular code in pathological cases as well? It seems unlikely to help with megamorphic call sites, because the compiled method handle has to be retrieved somehow, and the cost of that lookup would dwarf the gains.

But what about piecing together code at run time, and optimizing it? In particular, one could supply a data structure and an interpreter for that data structure, and “compile” them together using LambdaMetafactory. Would it be smart enough to partially evaluate the code given the data structure, and so turn your interpreter into the equivalent “plain” code?

Incidentally, that is exactly the approach taken by the Truffle framework, which runs on top of the Graal VM, so there is definitely something to the idea. Maybe something precludes it with the current JVM, hence requiring the GraalVM modification?

In any case, there is something to be said in favor of making these capabilities available as a library, which could be used in “regular programs” (i.e. not compilers). Writing a simple interpreter is often the easiest approach to some problems.

分析java进程假死情况

1 引言

 

1.1 编写目的

 

为了方便大家以后发现进程假死的时候能够正常的分析并且第一时间保留现场快照。

 

1.2编写背景

 

最近服务器发现tomcat的应用会偶尔出现无法访问的情况。经过一段时间的观察最近又发现有台tomcat的应用出现了无法访问情况。简单描述下该台tomcat当时具体的表现:客户端请求没有响应,查看服务器端tomcat的进程是存活的,查看业务日志的时候发现日志停止没有任何最新的访问日志。连tomcat下面的catalina.log也没有任何访问记录,基本断定该台tomcat已不能提供服务。

 

2 分析步骤

 

根据前面我描述的假死现象,我最先想到的是网络是否出现了问题,是不是有什么丢包严重的情况,于是我开始从请求的数据流程开始分析,由于我们业务的架构采用的是nginx+tomcat的集群配置,一个请求上来的流向可以用下图来简单的描述一下:

 

 

 

2.1检查nginx的网络情况

 

更改nginx的配置,让该台nginx请求只转到本机器的出现问题的tomcat应用上面,在access.log里看是否有网络请求,结果可以查看到当前所有的网络请求,也就是说可以排除是网络的问题。

 

2.2检查tomcat 的网络情况

 

分析业务配置的tomcat访问日志xxxx.log上是否有日志访问记录,经过查询该台tomcat应用日志完全没有任何访问记录,由于我们的部署是本机的nginx转到本机的tomcat应用,所以可以排除不是网络问题。到此基本可以断定网络没有问题,tomcat 本身出现了假死的情况。在tomcat的日志里有报过OutOfMemoryError的异常,所以可以肯定tomcat假死的原因是OOM

 

3 分析JVM内存溢出

 

3.1为什么会发生内存泄漏

 

在我们学习Java的时候就知道它最为方便的地方就是我们不需要管理内存的分配和释放,一切由JVM自己来进行处理,当Java对象不再被应用时,等到堆内存不够用时JVM会进行GC处理,清除这些对象占用的堆内存空间,但是如果对象一直被应用,那么JVM是无法对其进行GC处理的,那么我们创建新的对象时,JVM就没有办法从堆中获取足够的内存分配给此对象,这时就会导致OOM。我们出现OOM原因,一般都是因为我们不断的往容器里存放对象,然而容器没有相应的大小限制或清除机制,这样就容易导致OOM。

 

3.2快速定位问题

 

当我们的应用服务器占用了过多内存的时候,我们怎么样才能快速的定位问题呢?要想快速定位问题,首先我们必需获取服务器JVM某时刻的内存快照。Jdk里面提供了很多相应的命令比如:jstack,jstat,jmap,jps等等. 在出现问题后我们应该快速保留现场。

 

3.2.1 jstack

 

可以观察到jvm中当前所有线程的运行情况和线程当前状态.

 

sudo jstack -F 进程ID
输出内容如下:

从上面的图我们可以看到tomcat进程里面没有死锁的情况,而且每个线程都处理等待的状态。这个时候我们可以telnet命令连上tomcat的端口查看tomcat进程是否有任务回应。这时发现tomcat没有任何回应可以证明tomcat应用已没有响应处理假死状态。

 

3.2.2 jstat

这是jdk命令中比较重要,也是相当实用的一个命令,可以观察到classloader,compiler,gc相关信息
具体参数如下:
-class:统计class loader行为信息
-compile:统计编译行为信息
-gc:统计jdk gc时heap信息
-gccapacity:统计不同的generations(包括新生区,老年区,permanent区)相应的heap容量情况
-gccause:统计gc的情况,(同-gcutil)和引起gc的事件
-gcnew:统计gc时,新生代的情况
-gcnewcapacity:统计gc时,新生代heap容量
-gcold:统计gc时,老年区的情况
-gcoldcapacity:统计gc时,老年区heap容量
-gcpermcapacity:统计gc时,permanent区heap容量
-gcutil:统计gc时,heap情况
-printcompilation:不知道干什么的,一直没用过。

一般比较常用的几个参数是:
sudo jstat -class 2083 1000 10 (每隔1秒监控一次,一共做10次)

 

查看当时的head情况

 

sudo jstat -gcutil  20683 2000

注:该图不是出错截取

 

出现时候截取的数据是gc已经完全没有处理了,因为没有加上full gc的日志所以不确定JVM GC 时间过长,导致应用暂停.

 

3.2.3获取内存快照

 

Jdk自带的jmap可以获取内在某一时刻的快照

 

命令:jmap -dump:format=b,file=heap.bin <pid>
file:保存路径及文件名
pid:进程编号(windows通过任务管理器查看,linux通过ps aux查看)
dump文件可以通过MemoryAnalyzer分析查看,网址:http://www.eclipse.org/mat/,可以查看dump时对象数量,内存占用,线程情况等。

从上面的图可以看得出来对象没有内存溢出。

从上图我们可以明确的看出此项目的HashMap内存使用率比较高,因为我们的系统都是返回Map的数据结构所以占用比较高的内存是正常情况。

 

 

3.2.4观察运行中的jvm物理内存的占用情况

 

观察运行中的jvm物理内存的占用情况。我们也可以用jmap命令
参数如下:
-heap:打印jvm heap的情况
-histo:打印jvm heap的直方图。其输出信息包括类名,对象数量,对象占用大小。
-histo:live :同上,但是只答应存活对象的情况
-permstat:打印permanent generation heap情况

命令使用:
jmap -heap 2083
可以观察到New Generation(Eden Space,From Space,To Space),tenured generation,Perm Generation的内存使用情况
输出内容:

上图为tomcat应用出错前JVM的配置信息,可以明确的看到当时的信息:

 

MaxHeapSize堆内存大小为:3500M

 

MaxNewSize新生代内存大小:512M

 

PermSize永久代内存大小:192M

 

NewRatio设置年轻代(包括Eden和两个Survivor区)与年老代的比值(除去持久代)。设置为2,则年轻代与年老代所占比值为1:2,年轻代占整个堆栈的1/3

 

SurvivorRatio设置年轻代中Eden区与Survivor区的大小比值。设置为8,则两个Survivor区与一个Eden区的比值为2:8,一个Survivor区占整个年轻代的1/10

 

 

 

在New Generation中,有一个叫Eden的空间,主要是用来存放新生的对象,还有两个Survivor Spaces(from,to), 它们用来存放每次垃圾回收后存活下来的对象。在Old Generation中,主要存放应用程序中生命周期长的内存对象,还有个Permanent Generation,主要用来放JVM自己的反射对象,比如类对象和方法对象等。

 

 

 

 

从上面的图可以看出来JVM的新生代设置太小,可以看出应用的新生代区完全占满了,无法再往新生代区增加新的对象此时的这些对象都处于活跃状态,所以不会被GC处理,但是tomcat应用还在继续产生新的对象,这样就会导致OOM的发生,这就是导致tomcat假死的原因.

 

4 Tomcat假死其它情况

 

以下是网上资料说的tomcat假的情况:

 

1、应用本身程序的问题,造成死锁。

 

2、load 太高,已经超出服务的极限

 

3、jvm GC 时间过长,导致应用暂停

 

因为出错项目里面没有打出GC的处理情况,所以不确定此原因是否也是我项目tomcat假死的原因之一。

 

4、大量tcp 连接 CLOSE_WAIT

 

netstat -n | awk ‘/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}’

 

TIME_WAIT 48

 

CLOSE_WAIT 2228

 

ESTABLISHED 86

 

常用的三个状态是:ESTABLISHED 表示正在通信,TIME_WAIT 表示主动关闭,CLOSE_WAIT 表示被动关

ant build java generics,ANT编译泛型

用ant编译java项目的,报泛型错误,但eclipse却没有报错;报错如下:

ScopeUtil.java:185: type parameters of <T>T cannot be determined; no unique maximal instance exists for type variable T with upper bounds boolean,java.lang.Object

代码如下 : 

public static final boolean isSuperAdmin() {
   return getSession(key_SuperAdmin);
}

@SuppressWarnings(“unchecked”)
public static final <T> T getSession(String name) {
ActionContext actionContext = ActionContext.getContext();
Map<String, Object> session = actionContext.getSession();
return (T) session.get(name);
}

经过分析,发现isSuperAdmin返回是boolean而getSession,返回是Boolean,既返回对象与返回类型的区别,所以在ant编译时出错;

把代码改一下就可以了:

public static final boolean isSuperAdmin() {
Boolean superAdmin = getSession(key_SuperAdmin);
return superAdmin == true;
}

To solve the problem between JAXB2.1 and JDK1.6/6.0

Scenario 1
1.8. Using JAX-WS 2.1 with JavaSE6
JavaSE6 ships with JAX-WS 2.0 API in rt.jar, which causes some trouble when you try to run applications that use JAX-WS 2.1 API. This document collects information about how to solve this issue.

1.8.1. Endorsed directory
One way to fix this is to copy jaxws-api.jar and jaxb-api.jar into JRE endorsed directory, which is $JAVA_HOME/lib/endorsed (or $JDK_HOME/jre/lib/endorsed)

Some application containers, such as Glassfish, modifies the location of the endorsed directory to a different place. From inside the JVM, you can check the current location by doing System.out.println(System.getProperty(“java.endorsed.dirs”));

Obviously you still need other JAX-WS jars in your classpath.

Please do not put all the jars to the endorsed directory. This makes it impossible for JAX-WS RI to see other classes that it needs for its operation, such as servlet classes on the server-side, or Ant classes in the tool time. As those are not loaded by the bootstrap classloader, you’ll get NoClassDefError on servlet/Ant classes.

 

Scenario 2

Support for JDK 1.6
Jbossws 2.0.1.GA is based on the jax-ws and jaxb versions 2.1. But JDK 1.6 ships jaxb 2.0 classes as part of the core distribution (rt.jar). So in order for the jboss jars to take precedent over the jdk supplied jars, we have to use the endorsed directory mechanism, as described here. The above link is for using jaxws ri 2.1, but we need jbossws 2.0.1. For that purpose copy the following 3 jars in to the jboss/lib/endorsed directory.

So assuming your jboss is setup in a directory /jboss and you have already installed jbossws 2.0.1 on top of it, Copy the following 3 files to /jboss/lib/endorsed.

/jboss/server/default/lib/jboss-saaj.jar
/jboss/server/default/lib/jboss-jaxws.jar
/jboss/client/jaxb-api.jar
The jboss run script is already configured to add the /jboss/lib/endoresed directory to the list of endorsed directory. So jars in this directory will take precedence over rt.jar.