JBZipFile: allow adding entries without loading their content into memory

And use this in 'Package File' action to avoid FileTooBigException when updating big entries in archives (IDEA-109357).

java/compiler/impl/src/com/intellij/packaging/impl/ui/actions/PackageFileWorker.java

@@ -46,8 +46,7 @@ import com.intellij.util.io.zip.JBZipEntry;
 import com.intellij.util.io.zip.JBZipFile;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.File;
-import java.io.IOException;
+import java.io.*;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -89,6 +88,7 @@ public class PackageFileWorker {
                   packageFile(file, project, artifacts, packIntoArchives);
                 }
                 catch (IOException e) {
+                  LOG.info(e);
                   String message = CompilerBundle.message("message.tect.package.file.io.error", e.toString());
                   Notifications.Bus.notify(new Notification("Package File", "Cannot package file", message, NotificationType.ERROR));
                 }
@@ -167,7 +167,7 @@ public class PackageFileWorker {
       try {
         final String fullPathInArchive = DeploymentUtil.trimForwardSlashes(DeploymentUtil.appendToPath(pathInArchive, myRelativeOutputPath));
         final JBZipEntry entry = file.getOrCreateEntry(fullPathInArchive);
-        entry.setData(FileUtil.loadFileBytes(myFile));
+        entry.setDataFromFile(myFile);
       }
       finally {
         file.close();
@@ -186,10 +186,12 @@ public class PackageFileWorker {
         final File tempFile = FileUtil.createTempFile("packageFile" + FileUtil.sanitizeFileName(nextPathInArchive),
                                                       FileUtilRt.getExtension(PathUtil.getFileName(nextPathInArchive)));
         if (entry.getSize() != -1) {
-          FileUtil.writeToFile(tempFile, entry.getData());
+          try (OutputStream output = new BufferedOutputStream(new FileOutputStream(tempFile))) {
+            entry.writeDataTo(output);
+          }
         }
         packFile(FileUtil.toSystemIndependentName(tempFile.getAbsolutePath()), "", parentsTrail);
-        entry.setData(FileUtil.loadFileBytes(tempFile));
+        entry.setDataFromFile(tempFile);
         FileUtil.delete(tempFile);
       }
       finally {

platform/util/src/com/intellij/openapi/util/io/FileUtil.java

@@ -537,10 +537,14 @@ public class FileUtil extends FileUtilRt {
   }
 
   public static void copy(@NotNull InputStream inputStream, int maxSize, @NotNull OutputStream outputStream) throws IOException {
+    copy(inputStream, (long)maxSize, outputStream);
+  }
+
+  public static void copy(@NotNull InputStream inputStream, long maxSize, @NotNull OutputStream outputStream) throws IOException {
     final byte[] buffer = getThreadLocalBuffer();
-    int toRead = maxSize;
+    long toRead = maxSize;
     while (toRead > 0) {
-      int read = inputStream.read(buffer, 0, Math.min(buffer.length, toRead));
+      int read = inputStream.read(buffer, 0, (int)Math.min(buffer.length, toRead));
       if (read < 0) break;
       toRead -= read;
       outputStream.write(buffer, 0, read);

platform/util/src/com/intellij/util/io/zip/JBZipEntry.java

@@ -20,13 +20,11 @@
 package com.intellij.util.io.zip;
 
 import com.intellij.openapi.util.io.FileUtil;
+import com.intellij.openapi.util.io.FileUtilRt;
 import com.intellij.util.ArrayUtil;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.RandomAccessFile;
+import java.io.*;
 import java.util.zip.Inflater;
 import java.util.zip.InflaterInputStream;
 import java.util.zip.ZipEntry;
@@ -456,6 +454,33 @@ public class JBZipEntry implements Cloneable {
     setData(bytes, time);
   }
 
+  public void setDataFromFile(File file) throws IOException {
+    if (file.length() < FileUtilRt.LARGE_FOR_CONTENT_LOADING / 2) {
+      //for small files its faster to load their whole content into memory so we can write it to zip sequentially
+      setData(FileUtil.loadFileBytes(file));
+    }
+    else {
+      doSetDataFromFile(file);
+    }
+  }
+
+  void doSetDataFromFile(File file) throws IOException {
+    InputStream input = new BufferedInputStream(new FileInputStream(file));
+    try {
+      myFile.getOutputStream().putNextEntryContent(this, file.length(), input);
+    }
+    finally {
+      input.close();
+    }
+  }
+
+  public void writeDataTo(OutputStream output) throws IOException {
+    if (size == -1) throw new IOException("no data");
+
+    InputStream stream = getInputStream();
+    FileUtil.copy(stream, (int)size, output);
+  }
+
   public byte[] getData() throws IOException {
     if (size == -1) throw new IOException("no data");
 

platform/util/src/com/intellij/util/io/zip/JBZipFile.java

@@ -411,21 +411,25 @@ public class JBZipFile {
   }
 
   /**
-   * Number of bytes in local file header up to the "length of
-   * filename" entry.
+   * Number of bytes in local file header up to the "crc" entry.
    */
-  static final long LFH_OFFSET_FOR_FILENAME_LENGTH =
+  static final long LFH_OFFSET_FOR_CRC =
     /* local file header signature     */ WORD
                                           /* version needed to extract       */ + SHORT
                                           /* general purpose bit flag        */ + SHORT
                                           /* compression method              */ + SHORT
                                           /* last mod file time              */ + SHORT
-                                          /* last mod file date              */ + SHORT
+                                          /* last mod file date              */ + SHORT;
+
+  /**
+   * Number of bytes in local file header up to the "length of filename" entry.
+   */
+  static final long LFH_OFFSET_FOR_FILENAME_LENGTH =
+                                          LFH_OFFSET_FOR_CRC
                                           /* crc-32                          */ + WORD
                                           /* compressed size                 */ + WORD
                                           /* uncompressed size               */ + WORD;
 
-
   /**
    * Retrieve a String from the given bytes using the encoding set
    * for this ZipFile.

platform/util/src/com/intellij/util/io/zip/JBZipOutputStream.java

@@ -21,10 +21,9 @@ package com.intellij.util.io.zip;
 
 
 import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream;
+import org.jetbrains.annotations.NotNull;
 
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.io.UnsupportedEncodingException;
+import java.io.*;
 import java.util.List;
 import java.util.zip.*;
 
@@ -237,6 +236,16 @@ class JBZipOutputStream {
     writeOut(extra);
   }
 
+  private void updateLocalFileHeader(JBZipEntry ze, long crc, long compressedSize) throws IOException {
+    ze.setCrc(crc);
+    ze.setCompressedSize(compressedSize);
+    flushBuffer();
+    long offset = ze.getHeaderOffset() + JBZipFile.LFH_OFFSET_FOR_CRC;
+    raf.seek(offset);
+    raf.write(ZipLong.getBytes(crc));
+    raf.write(ZipLong.getBytes(compressedSize));
+  }
+
   private void writeOutShort(int s) throws IOException {
     writeOut(ZipShort.getBytes(s));
   }
@@ -389,20 +398,12 @@ class JBZipOutputStream {
   }
 
   public void putNextEntryBytes(JBZipEntry entry, byte[] bytes) throws IOException {
-    entry.setSize(bytes.length);
+    prepareNextEntry(entry, bytes.length);
 
     crc.reset();
     crc.update(bytes);
     entry.setCrc(crc.getValue());
 
-    if (entry.getMethod() == -1) {
-      entry.setMethod(method);
-    }
-
-    if (entry.getTime() == -1) {
-      entry.setTime(System.currentTimeMillis());
-    }
-
     final byte[] outputBytes;
     final int outputBytesLength;
     if (entry.getMethod() == ZipEntry.DEFLATED) {
@@ -428,7 +429,74 @@ class JBZipOutputStream {
     writeOut(outputBytes, 0, outputBytesLength);
   }
 
+  void putNextEntryContent(JBZipEntry entry, long size, InputStream content) throws IOException {
+    prepareNextEntry(entry, size);
+    writeLocalFileHeader(entry);
+    flushBuffer();
+
+    RandomAccessFileOutputStream fileOutput = new RandomAccessFileOutputStream(raf);
+    OutputStream bufferedFileOutput = new BufferedOutputStream(fileOutput);
+
+    OutputStream output;
+    if (entry.getMethod() == ZipEntry.DEFLATED) {
+      def.setLevel(level);
+      output = new DeflaterOutputStream(bufferedFileOutput, def);
+    }
+    else {
+      output = bufferedFileOutput;
+    }
+
+    try {
+      final byte[] buffer = new byte[10 * 1024];
+      int count;
+      crc.reset();
+      while ((count = content.read(buffer)) > 0) {
+        output.write(buffer, 0, count);
+        crc.update(buffer, 0, count);
+      }
+    }
+    finally {
+      output.close();
+    }
+    writtenOnDisk += fileOutput.myWrittenBytes;
+
+    updateLocalFileHeader(entry, crc.getValue(), fileOutput.myWrittenBytes);
+  }
+
+  private void prepareNextEntry(JBZipEntry entry, long size) {
+    entry.setSize(size);
+
+    if (entry.getMethod() == -1) {
+      entry.setMethod(method);
+    }
+
+    if (entry.getTime() == -1) {
+      entry.setTime(System.currentTimeMillis());
+    }
+  }
+
   long getWritten() {
     return writtenOnDisk + myBuffer.size();
   }
+
+  private static class RandomAccessFileOutputStream extends OutputStream {
+    private final RandomAccessFile myFile;
+    private long myWrittenBytes;
+
+    public RandomAccessFileOutputStream(RandomAccessFile file) {
+      myFile = file;
+    }
+
+    @Override
+    public void write(int b) throws IOException {
+      myFile.write(b);
+      myWrittenBytes++;
+    }
+
+    @Override
+    public void write(@NotNull byte[] b, int off, int len) throws IOException {
+      myFile.write(b, off, len);
+      myWrittenBytes += len;
+    }
+  }
 }

platform/util/testSrc/com/intellij/util/io/zip/UpdateZipFromFileTest.kt

+// Copyright 2000-2017 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.io.zip
+
+import com.intellij.util.io.directoryContent
+import org.junit.Test
+import java.io.File
+
+/**
+ * @author nik
+ */
+class UpdateZipFromFileTest {
+  @Test
+  fun `add entry`() {
+    val dir = directoryContent {
+      zip("a.zip") {
+        file("a.txt", text = "a")
+      }
+      file("b.txt", text = "b")
+    }.generateInTempDir()
+
+    val zip = JBZipFile(File(dir, "a.zip"))
+    zip.getOrCreateEntry("b.txt").setDataFromFile(File(dir, "b.txt"))
+    zip.close()
+
+    directoryContent {
+      zip("a.zip") {
+        file("a.txt", text = "a")
+        file("b.txt", text = "b")
+      }
+      file("b.txt", text = "b")
+    }
+  }
+
+  @Test
+  fun `replace entry`() {
+    val dir = directoryContent {
+      zip("a.zip") {
+        file("a.txt", text = "a")
+      }
+      file("b.txt", text = "b")
+    }.generateInTempDir()
+
+    val zip = JBZipFile(File(dir, "a.zip"))
+    zip.getOrCreateEntry("a.txt").setDataFromFile(File(dir, "b.txt"))
+    zip.close()
+
+    directoryContent {
+      zip("a.zip") {
+        file("a.txt", text = "b")
+      }
+      file("b.txt", text = "b")
+    }
+  }
+}
\ No newline at end of file