Posted By

kubaitis on 11/01/10


Tagged

files gzip compression


Versions (?)

Who likes this?

2 people have marked this snippet as a favorite

kubaitis
cruelfate


Convenience class for working with Gzip text files.


 / Published in: Java
 

I wrote this for an enterprise level event logging cluster that has more traffic than Oprah. Has worked great for years of uptime... wish java had a decent bz2 implementation though.

  1. import java.io.*;
  2. import java.util.zip.GZIPInputStream;
  3. import java.util.zip.GZIPOutputStream;
  4. import java.util.Random;
  5. import java.nio.channels.FileChannel;
  6. import java.nio.channels.FileLock;
  7. import java.nio.channels.OverlappingFileLockException;
  8. public class GZ implements Serializable {
  9. private boolean makeDirectories(final String path){
  10. boolean result;
  11. try{
  12. (new File(path).getParentFile()).mkdirs();
  13. result=true;
  14. }catch(Exception e){
  15. result=false;
  16. e.printStackTrace();
  17. }
  18. return result;
  19. }
  20. private String getParent(final String path){
  21. StringBuilder result = new StringBuilder();
  22. try{
  23. result.append(new File(path).getParent());
  24. if(!result.toString().endsWith("/")){result.append("/");}
  25. }catch(Exception e){ e.printStackTrace(); }
  26. return result.toString();
  27. }
  28. public StringBuilder readTextFile(final String filename){
  29. StringBuilder sb = new StringBuilder();
  30. FileInputStream fin = null;
  31. GZIPInputStream gzis = null;
  32. InputStreamReader xover = null;
  33. BufferedReader is = null;
  34. FileChannel channel = null;
  35. try{
  36. File file = new File(filename);
  37. if(file.exists()){
  38. channel = new RandomAccessFile(file, "rw").getChannel();
  39. FileLock lock = null;
  40. int attempt = 0;
  41. while (lock==null && attempt < 5){
  42. if(attempt > 0 ){Thread.sleep(1000);}
  43. try{lock=channel.tryLock();}catch(OverlappingFileLockException ofe){}
  44. attempt++;
  45. }
  46. if(lock !=null){
  47. fin = new FileInputStream(file);
  48. gzis = new GZIPInputStream(fin);
  49. xover = new InputStreamReader(gzis);
  50. is = new BufferedReader(xover);
  51. String line;
  52. while ((line = is.readLine( )) != null)sb.append(line).append("\n");
  53. lock.release();
  54. }
  55. }
  56. }catch(EOFException e){
  57. System.out.println(new StringBuilder("EOF: (probably minor) ").append(filename).toString());
  58. e.printStackTrace();
  59. }catch(Exception e){
  60. System.out.println(new StringBuilder("Bug GZ.readTextFile(): ").append(filename).toString());
  61. e.printStackTrace();
  62. }finally{
  63. if(channel!=null){try{channel.close();}catch(Exception e){}}
  64. if(is!=null){try{is.close();}catch(Exception e){}}
  65. if(xover!=null){try{xover.close();}catch(Exception e){}}
  66. if(gzis!=null){try{gzis.close();}catch(Exception e){}}
  67. if(fin!=null){try{fin.close();}catch(Exception e){}}
  68. }
  69. return sb;
  70. }
  71. public boolean writeTextFile(final String filename,final StringBuilder content){
  72. boolean result=false;
  73. FileChannel channel=null;
  74. GZIPOutputStream out=null;
  75. makeDirectories(filename);
  76. try {channel=new RandomAccessFile(filename,"rw").getChannel();
  77. FileLock lock = null;
  78. int attempt = 0;
  79. while (lock==null && attempt < 5){
  80. if(attempt > 0 ){Thread.sleep(1000);}
  81. try{lock=channel.tryLock();}catch(OverlappingFileLockException ofe){}
  82. attempt++;
  83. }
  84. if(lock !=null){
  85. out=new GZIPOutputStream(new FileOutputStream(filename));
  86. out.write(content.toString().getBytes());
  87. lock.release();
  88. result=true;
  89. }
  90. }catch(Exception e){
  91. System.out.println(new StringBuilder("Bug GZ.writeTextFile(): ").append(filename).toString());
  92. e.printStackTrace();
  93. }finally{
  94. try{if(channel!=null){channel.close();}}catch(Exception e){}
  95. try{if(out!=null){out.close();}}catch(Exception e){}
  96. }
  97. return result;
  98. }
  99. public boolean appendTextfile(final String filename,final StringBuilder content){
  100. boolean result=false;
  101. FileInputStream readIn = null;
  102. GZIPInputStream readGZ = null;
  103. InputStreamReader readISR = null;
  104. BufferedReader readBR = null;
  105. FileChannel readChannel = null;
  106. FileChannel tempChannel=null;
  107. GZIPOutputStream tempGZ=null;
  108. Random r = new Random();
  109. FileLock readLock=null;
  110. FileLock tempLock=null;
  111. // USE TEMP FILE
  112. makeDirectories(filename);
  113. String tempname = new StringBuilder(getParent(filename)).append("temp_").append(r.nextInt(9999999)).toString();
  114. File readFile=new File(filename);
  115. File tempFile=new File(tempname);
  116. try{
  117. tempChannel=new RandomAccessFile(tempFile,"rw").getChannel();
  118. int attempt=0;
  119. while(tempLock==null&&attempt<5){if(attempt>0){Thread.sleep(1000);}try{tempLock=tempChannel.tryLock();}catch(OverlappingFileLockException ofe){attempt++;}}
  120. if(tempLock!=null){tempGZ=new GZIPOutputStream(new FileOutputStream(tempFile));}
  121. // COPY PREVIOUS RECORDS TO TEMP FILE
  122. if(readFile.exists()){
  123. readChannel=new RandomAccessFile(readFile, "rw").getChannel();
  124. int attemptA=0;
  125. while(readLock==null&&attemptA<5){
  126. if(attemptA>0){Thread.sleep(1000);}
  127. try{readLock=readChannel.tryLock();}catch(OverlappingFileLockException ofe){}
  128. attemptA++;
  129. }
  130. if(readLock!=null){
  131. readIn=new FileInputStream(readFile);
  132. readGZ=new GZIPInputStream(readIn);
  133. readISR=new InputStreamReader(readGZ);
  134. readBR=new BufferedReader(readISR);
  135. String line;
  136. while((line=readBR.readLine())!=null){if(tempLock!=null)tempGZ.write(new StringBuilder(line.trim()).append("\n").toString().getBytes());}
  137. }
  138. }
  139. // WRITE NEW DATA TO TEMPFILE
  140. if(tempLock!=null){
  141. if(content.charAt(content.length()-1)!='\n'){content.append("\n");}
  142. tempGZ.write(content.toString().getBytes());
  143. tempGZ.finish();
  144. tempGZ.close();
  145. }
  146. // RENAME THE TEMP FILE TO THE OLDFILE
  147. boolean success = tempFile.renameTo(readFile);
  148. if(success){tempFile.delete();result=true;}else{System.out.println("HUGE PROBLEM... could not rename file:" + readFile.getAbsolutePath());result=false;}
  149. // RELEASE THE FILE LOCKS
  150. if(tempLock!=null){tempLock.release();}
  151. if(readLock!=null){readLock.release();}
  152. }catch(EOFException e){System.out.println(new StringBuilder("EOF: (probably minor) ").append(filename).toString());e.printStackTrace();
  153. }catch(Exception e){System.out.println(new StringBuilder("Bug GZ.appendTextFile(): ").append(filename).toString());e.printStackTrace();
  154. }finally{
  155. if(readChannel!=null){try{readChannel.close();}catch(Exception e){}}
  156. if(readIn!=null){try{readIn.close();}catch(Exception e){}}
  157. if(readISR!=null){try{readISR.close();}catch(Exception e){}}
  158. if(readGZ!=null){try{readGZ.close();}catch(Exception e){}}
  159. if(readBR!=null){try{readBR.close();}catch(Exception e){}}
  160. if(tempChannel!=null){try{tempChannel.close();}catch(Exception e){}}
  161. if(tempGZ!=null){try{tempGZ.close();}catch(Exception e){}}
  162. }
  163. return result;
  164. }
  165. public void renameFile(final String old_name,final String new_name){
  166. File f = new File(old_name);
  167. f.renameTo(new File(new_name));
  168. }
  169. public boolean prependTextFile(final String filename,final StringBuilder content){
  170. boolean result=false;
  171. StringBuilder previous = null;
  172. try{ previous = readTextFile(filename);}catch(Exception e){/* Did not previously exist */}
  173. FileChannel channel=null;
  174. GZIPOutputStream out=null;
  175. makeDirectories(filename);
  176. try {channel=new RandomAccessFile(filename,"rw").getChannel();
  177. FileLock lock = null;
  178. int attempt = 0;
  179. while (lock==null && attempt < 5){
  180. if(attempt > 0 ){Thread.sleep(1000);}
  181. try{lock=channel.tryLock();}catch(OverlappingFileLockException ofe){}
  182. attempt++;
  183. }
  184. if(lock !=null){
  185. out=new GZIPOutputStream(new FileOutputStream(filename));
  186. out.write(content.toString().getBytes());
  187. if(previous!=null){out.write(previous.toString().getBytes());}
  188. lock.release();
  189. result=true;
  190. }
  191. }catch(Exception e){
  192. System.out.println(new StringBuilder("Bug GZ.prependTextFile(): ").append(filename).toString());
  193. e.printStackTrace();
  194. }
  195. finally{
  196. try{if(channel!=null){channel.close();}}catch(Exception e){}
  197. try{if(out!=null){out.close();}}catch(Exception e){}
  198. }
  199. return result;
  200. }
  201. }

Report this snippet  

You need to login to post a comment.