| /* |
| * Copyright (C) 2008-2009, Google Inc. |
| * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> |
| * and other copyright owners as documented in the project's IP log. |
| * |
| * This program and the accompanying materials are made available |
| * under the terms of the Eclipse Distribution License v1.0 which |
| * accompanies this distribution, is reproduced below, and is |
| * available at http://www.eclipse.org/org/documents/edl-v10.php |
| * |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or |
| * without modification, are permitted provided that the following |
| * conditions are met: |
| * |
| * - Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * |
| * - Redistributions in binary form must reproduce the above |
| * copyright notice, this list of conditions and the following |
| * disclaimer in the documentation and/or other materials provided |
| * with the distribution. |
| * |
| * - Neither the name of the Eclipse Foundation, Inc. nor the |
| * names of its contributors may be used to endorse or promote |
| * products derived from this software without specific prior |
| * written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
| * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
| * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
| * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
| * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| package org.eclipse.jgit.util; |
| |
| import java.io.BufferedOutputStream; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileOutputStream; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.OutputStream; |
| import java.util.ArrayList; |
| |
| import org.eclipse.jgit.internal.JGitText; |
| import org.eclipse.jgit.lib.NullProgressMonitor; |
| import org.eclipse.jgit.lib.ProgressMonitor; |
| |
| /** |
| * A fully buffered output stream. |
| * <p> |
| * Subclasses determine the behavior when the in-memory buffer capacity has been |
| * exceeded and additional bytes are still being received for output. |
| */ |
| public abstract class TemporaryBuffer extends OutputStream { |
| /** Default limit for in-core storage. */ |
| protected static final int DEFAULT_IN_CORE_LIMIT = 1024 * 1024; |
| |
| /** Chain of data, if we are still completely in-core; otherwise null. */ |
| ArrayList<Block> blocks; |
| |
| /** |
| * Maximum number of bytes we will permit storing in memory. |
| * <p> |
| * When this limit is reached the data will be shifted to a file on disk, |
| * preventing the JVM heap from growing out of control. |
| */ |
| private int inCoreLimit; |
| |
| /** Initial size of block list. */ |
| private int initialBlocks; |
| |
| /** If {@link #inCoreLimit} has been reached, remainder goes here. */ |
| private OutputStream overflow; |
| |
| /** |
| * Create a new empty temporary buffer. |
| * |
| * @param limit |
| * maximum number of bytes to store in memory before entering the |
| * overflow output path; also used as the estimated size. |
| */ |
| protected TemporaryBuffer(int limit) { |
| this(limit, limit); |
| } |
| |
| /** |
| * Create a new empty temporary buffer. |
| * |
| * @param estimatedSize |
| * estimated size of storage used, to size the initial list of |
| * block pointers. |
| * @param limit |
| * maximum number of bytes to store in memory before entering the |
| * overflow output path. |
| * @since 4.0 |
| */ |
| protected TemporaryBuffer(int estimatedSize, int limit) { |
| if (estimatedSize > limit) |
| throw new IllegalArgumentException(); |
| this.inCoreLimit = limit; |
| this.initialBlocks = (estimatedSize - 1) / Block.SZ + 1; |
| reset(); |
| } |
| |
| /** {@inheritDoc} */ |
| @Override |
| public void write(int b) throws IOException { |
| if (overflow != null) { |
| overflow.write(b); |
| return; |
| } |
| |
| Block s = last(); |
| if (s.isFull()) { |
| if (reachedInCoreLimit()) { |
| overflow.write(b); |
| return; |
| } |
| |
| s = new Block(); |
| blocks.add(s); |
| } |
| s.buffer[s.count++] = (byte) b; |
| } |
| |
| /** {@inheritDoc} */ |
| @Override |
| public void write(byte[] b, int off, int len) throws IOException { |
| if (overflow == null) { |
| while (len > 0) { |
| Block s = last(); |
| if (s.isFull()) { |
| if (reachedInCoreLimit()) |
| break; |
| |
| s = new Block(); |
| blocks.add(s); |
| } |
| |
| final int n = Math.min(s.buffer.length - s.count, len); |
| System.arraycopy(b, off, s.buffer, s.count, n); |
| s.count += n; |
| len -= n; |
| off += n; |
| } |
| } |
| |
| if (len > 0) |
| overflow.write(b, off, len); |
| } |
| |
| /** |
| * Dumps the entire buffer into the overflow stream, and flushes it. |
| * |
| * @throws java.io.IOException |
| * the overflow stream cannot be started, or the buffer contents |
| * cannot be written to it, or it failed to flush. |
| */ |
| protected void doFlush() throws IOException { |
| if (overflow == null) |
| switchToOverflow(); |
| overflow.flush(); |
| } |
| |
| /** |
| * Copy all bytes remaining on the input stream into this buffer. |
| * |
| * @param in |
| * the stream to read from, until EOF is reached. |
| * @throws java.io.IOException |
| * an error occurred reading from the input stream, or while |
| * writing to a local temporary file. |
| */ |
| public void copy(InputStream in) throws IOException { |
| if (blocks != null) { |
| for (;;) { |
| Block s = last(); |
| if (s.isFull()) { |
| if (reachedInCoreLimit()) |
| break; |
| s = new Block(); |
| blocks.add(s); |
| } |
| |
| int n = in.read(s.buffer, s.count, s.buffer.length - s.count); |
| if (n < 1) |
| return; |
| s.count += n; |
| } |
| } |
| |
| final byte[] tmp = new byte[Block.SZ]; |
| int n; |
| while ((n = in.read(tmp)) > 0) |
| overflow.write(tmp, 0, n); |
| } |
| |
| /** |
| * Obtain the length (in bytes) of the buffer. |
| * <p> |
| * The length is only accurate after {@link #close()} has been invoked. |
| * |
| * @return total length of the buffer, in bytes. |
| */ |
| public long length() { |
| return inCoreLength(); |
| } |
| |
| private long inCoreLength() { |
| final Block last = last(); |
| return ((long) blocks.size() - 1) * Block.SZ + last.count; |
| } |
| |
| /** |
| * Convert this buffer's contents into a contiguous byte array. |
| * <p> |
| * The buffer is only complete after {@link #close()} has been invoked. |
| * |
| * @return the complete byte array; length matches {@link #length()}. |
| * @throws java.io.IOException |
| * an error occurred reading from a local temporary file |
| */ |
| public byte[] toByteArray() throws IOException { |
| final long len = length(); |
| if (Integer.MAX_VALUE < len) |
| throw new OutOfMemoryError(JGitText.get().lengthExceedsMaximumArraySize); |
| final byte[] out = new byte[(int) len]; |
| int outPtr = 0; |
| for (Block b : blocks) { |
| System.arraycopy(b.buffer, 0, out, outPtr, b.count); |
| outPtr += b.count; |
| } |
| return out; |
| } |
| |
| /** |
| * Convert this buffer's contents into a contiguous byte array. If this size |
| * of the buffer exceeds the limit only return the first {@code limit} bytes |
| * <p> |
| * The buffer is only complete after {@link #close()} has been invoked. |
| * |
| * @param limit |
| * the maximum number of bytes to be returned |
| * @return the byte array limited to {@code limit} bytes. |
| * @throws java.io.IOException |
| * an error occurred reading from a local temporary file |
| * @since 4.2 |
| */ |
| public byte[] toByteArray(int limit) throws IOException { |
| final long len = Math.min(length(), limit); |
| if (Integer.MAX_VALUE < len) |
| throw new OutOfMemoryError( |
| JGitText.get().lengthExceedsMaximumArraySize); |
| final byte[] out = new byte[(int) len]; |
| int outPtr = 0; |
| for (Block b : blocks) { |
| System.arraycopy(b.buffer, 0, out, outPtr, b.count); |
| outPtr += b.count; |
| } |
| return out; |
| } |
| |
| /** |
| * Send this buffer to an output stream. |
| * <p> |
| * This method may only be invoked after {@link #close()} has completed |
| * normally, to ensure all data is completely transferred. |
| * |
| * @param os |
| * stream to send this buffer's complete content to. |
| * @param pm |
| * if not null progress updates are sent here. Caller should |
| * initialize the task and the number of work units to <code> |
| * {@link #length()}/1024</code>. |
| * @throws java.io.IOException |
| * an error occurred reading from a temporary file on the local |
| * system, or writing to the output stream. |
| */ |
| public void writeTo(OutputStream os, ProgressMonitor pm) |
| throws IOException { |
| if (pm == null) |
| pm = NullProgressMonitor.INSTANCE; |
| for (Block b : blocks) { |
| os.write(b.buffer, 0, b.count); |
| pm.update(b.count / 1024); |
| } |
| } |
| |
| /** |
| * Open an input stream to read from the buffered data. |
| * <p> |
| * This method may only be invoked after {@link #close()} has completed |
| * normally, to ensure all data is completely transferred. |
| * |
| * @return a stream to read from the buffer. The caller must close the |
| * stream when it is no longer useful. |
| * @throws java.io.IOException |
| * an error occurred opening the temporary file. |
| */ |
| public InputStream openInputStream() throws IOException { |
| return new BlockInputStream(); |
| } |
| |
| /** |
| * Same as {@link #openInputStream()} but handling destruction of any |
| * associated resources automatically when closing the returned stream. |
| * |
| * @return an InputStream which will automatically destroy any associated |
| * temporary file on {@link #close()} |
| * @throws IOException |
| * in case of an error. |
| * @since 4.11 |
| */ |
| public InputStream openInputStreamWithAutoDestroy() throws IOException { |
| return new BlockInputStream() { |
| @Override |
| public void close() throws IOException { |
| super.close(); |
| destroy(); |
| } |
| }; |
| } |
| |
| /** |
| * Reset this buffer for reuse, purging all buffered content. |
| */ |
| public void reset() { |
| if (overflow != null) { |
| destroy(); |
| } |
| if (blocks != null) |
| blocks.clear(); |
| else |
| blocks = new ArrayList<>(initialBlocks); |
| blocks.add(new Block(Math.min(inCoreLimit, Block.SZ))); |
| } |
| |
| /** |
| * Open the overflow output stream, so the remaining output can be stored. |
| * |
| * @return the output stream to receive the buffered content, followed by |
| * the remaining output. |
| * @throws java.io.IOException |
| * the buffer cannot create the overflow stream. |
| */ |
| protected abstract OutputStream overflow() throws IOException; |
| |
| private Block last() { |
| return blocks.get(blocks.size() - 1); |
| } |
| |
| private boolean reachedInCoreLimit() throws IOException { |
| if (inCoreLength() < inCoreLimit) |
| return false; |
| |
| switchToOverflow(); |
| return true; |
| } |
| |
| private void switchToOverflow() throws IOException { |
| overflow = overflow(); |
| |
| final Block last = blocks.remove(blocks.size() - 1); |
| for (Block b : blocks) |
| overflow.write(b.buffer, 0, b.count); |
| blocks = null; |
| |
| overflow = new BufferedOutputStream(overflow, Block.SZ); |
| overflow.write(last.buffer, 0, last.count); |
| } |
| |
| /** {@inheritDoc} */ |
| @Override |
| public void close() throws IOException { |
| if (overflow != null) { |
| try { |
| overflow.close(); |
| } finally { |
| overflow = null; |
| } |
| } |
| } |
| |
| /** |
| * Clear this buffer so it has no data, and cannot be used again. |
| */ |
| public void destroy() { |
| blocks = null; |
| |
| if (overflow != null) { |
| try { |
| overflow.close(); |
| } catch (IOException err) { |
| // We shouldn't encounter an error closing the file. |
| } finally { |
| overflow = null; |
| } |
| } |
| } |
| |
| /** |
| * A fully buffered output stream using local disk storage for large data. |
| * <p> |
| * Initially this output stream buffers to memory and is therefore similar |
| * to ByteArrayOutputStream, but it shifts to using an on disk temporary |
| * file if the output gets too large. |
| * <p> |
| * The content of this buffered stream may be sent to another OutputStream |
| * only after this stream has been properly closed by {@link #close()}. |
| */ |
| public static class LocalFile extends TemporaryBuffer { |
| /** Directory to store the temporary file under. */ |
| private final File directory; |
| |
| /** |
| * Location of our temporary file if we are on disk; otherwise null. |
| * <p> |
| * If we exceeded the {@link #inCoreLimit} we nulled out {@link #blocks} |
| * and created this file instead. All output goes here through |
| * {@link #overflow}. |
| */ |
| private File onDiskFile; |
| |
| /** |
| * Create a new temporary buffer, limiting memory usage. |
| * |
| * @param directory |
| * if the buffer has to spill over into a temporary file, the |
| * directory where the file should be saved. If null the |
| * system default temporary directory (for example /tmp) will |
| * be used instead. |
| */ |
| public LocalFile(File directory) { |
| this(directory, DEFAULT_IN_CORE_LIMIT); |
| } |
| |
| /** |
| * Create a new temporary buffer, limiting memory usage. |
| * |
| * @param directory |
| * if the buffer has to spill over into a temporary file, the |
| * directory where the file should be saved. If null the |
| * system default temporary directory (for example /tmp) will |
| * be used instead. |
| * @param inCoreLimit |
| * maximum number of bytes to store in memory. Storage beyond |
| * this limit will use the local file. |
| */ |
| public LocalFile(File directory, int inCoreLimit) { |
| super(inCoreLimit); |
| this.directory = directory; |
| } |
| |
| @Override |
| protected OutputStream overflow() throws IOException { |
| onDiskFile = File.createTempFile("jgit_", ".buf", directory); //$NON-NLS-1$ //$NON-NLS-2$ |
| return new BufferedOutputStream(new FileOutputStream(onDiskFile)); |
| } |
| |
| @Override |
| public long length() { |
| if (onDiskFile == null) { |
| return super.length(); |
| } |
| return onDiskFile.length(); |
| } |
| |
| @Override |
| public byte[] toByteArray() throws IOException { |
| if (onDiskFile == null) { |
| return super.toByteArray(); |
| } |
| |
| final long len = length(); |
| if (Integer.MAX_VALUE < len) |
| throw new OutOfMemoryError(JGitText.get().lengthExceedsMaximumArraySize); |
| final byte[] out = new byte[(int) len]; |
| try (FileInputStream in = new FileInputStream(onDiskFile)) { |
| IO.readFully(in, out, 0, (int) len); |
| } |
| return out; |
| } |
| |
| @Override |
| public void writeTo(OutputStream os, ProgressMonitor pm) |
| throws IOException { |
| if (onDiskFile == null) { |
| super.writeTo(os, pm); |
| return; |
| } |
| if (pm == null) |
| pm = NullProgressMonitor.INSTANCE; |
| try (FileInputStream in = new FileInputStream(onDiskFile)) { |
| int cnt; |
| final byte[] buf = new byte[Block.SZ]; |
| while ((cnt = in.read(buf)) >= 0) { |
| os.write(buf, 0, cnt); |
| pm.update(cnt / 1024); |
| } |
| } |
| } |
| |
| @Override |
| public InputStream openInputStream() throws IOException { |
| if (onDiskFile == null) |
| return super.openInputStream(); |
| return new FileInputStream(onDiskFile); |
| } |
| |
| @Override |
| public InputStream openInputStreamWithAutoDestroy() throws IOException { |
| if (onDiskFile == null) { |
| return super.openInputStreamWithAutoDestroy(); |
| } |
| return new FileInputStream(onDiskFile) { |
| @Override |
| public void close() throws IOException { |
| super.close(); |
| destroy(); |
| } |
| }; |
| } |
| |
| @Override |
| public void destroy() { |
| super.destroy(); |
| |
| if (onDiskFile != null) { |
| try { |
| if (!onDiskFile.delete()) |
| onDiskFile.deleteOnExit(); |
| } finally { |
| onDiskFile = null; |
| } |
| } |
| } |
| } |
| |
| /** |
| * A temporary buffer that will never exceed its in-memory limit. |
| * <p> |
| * If the in-memory limit is reached an IOException is thrown, rather than |
| * attempting to spool to local disk. |
| */ |
| public static class Heap extends TemporaryBuffer { |
| /** |
| * Create a new heap buffer with a maximum storage limit. |
| * |
| * @param limit |
| * maximum number of bytes that can be stored in this buffer; |
| * also used as the estimated size. Storing beyond this many |
| * will cause an IOException to be thrown during write. |
| */ |
| public Heap(int limit) { |
| super(limit); |
| } |
| |
| /** |
| * Create a new heap buffer with a maximum storage limit. |
| * |
| * @param estimatedSize |
| * estimated size of storage used, to size the initial list of |
| * block pointers. |
| * @param limit |
| * maximum number of bytes that can be stored in this buffer. |
| * Storing beyond this many will cause an IOException to be |
| * thrown during write. |
| * @since 4.0 |
| */ |
| public Heap(int estimatedSize, int limit) { |
| super(estimatedSize, limit); |
| } |
| |
| @Override |
| protected OutputStream overflow() throws IOException { |
| throw new IOException(JGitText.get().inMemoryBufferLimitExceeded); |
| } |
| } |
| |
| static class Block { |
| static final int SZ = 8 * 1024; |
| |
| final byte[] buffer; |
| |
| int count; |
| |
| Block() { |
| buffer = new byte[SZ]; |
| } |
| |
| Block(int sz) { |
| buffer = new byte[sz]; |
| } |
| |
| boolean isFull() { |
| return count == buffer.length; |
| } |
| } |
| |
| private class BlockInputStream extends InputStream { |
| private byte[] singleByteBuffer; |
| private int blockIndex; |
| private Block block; |
| private int blockPos; |
| |
| BlockInputStream() { |
| block = blocks.get(blockIndex); |
| } |
| |
| @Override |
| public int read() throws IOException { |
| if (singleByteBuffer == null) |
| singleByteBuffer = new byte[1]; |
| int n = read(singleByteBuffer); |
| return n == 1 ? singleByteBuffer[0] & 0xff : -1; |
| } |
| |
| @Override |
| public long skip(long cnt) throws IOException { |
| long skipped = 0; |
| while (0 < cnt) { |
| int n = (int) Math.min(block.count - blockPos, cnt); |
| if (0 < n) { |
| blockPos += n; |
| skipped += n; |
| cnt -= n; |
| } else if (nextBlock()) |
| continue; |
| else |
| break; |
| } |
| return skipped; |
| } |
| |
| @Override |
| public int read(byte[] b, int off, int len) throws IOException { |
| if (len == 0) |
| return 0; |
| int copied = 0; |
| while (0 < len) { |
| int c = Math.min(block.count - blockPos, len); |
| if (0 < c) { |
| System.arraycopy(block.buffer, blockPos, b, off, c); |
| blockPos += c; |
| off += c; |
| len -= c; |
| copied += c; |
| } else if (nextBlock()) |
| continue; |
| else |
| break; |
| } |
| return 0 < copied ? copied : -1; |
| } |
| |
| private boolean nextBlock() { |
| if (++blockIndex < blocks.size()) { |
| block = blocks.get(blockIndex); |
| blockPos = 0; |
| return true; |
| } |
| return false; |
| } |
| } |
| } |