src/main/java/com/gitblit/manager/FilestoreManager.java - gitblit - Git at Google

 /*
  * Copyright 2015 gitblit.com.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.gitblit.manager;

 import java.io.EOFException;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.RandomAccessFile;
 import java.lang.reflect.Type;
 import java.nio.file.Files;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Pattern;

 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 import com.gitblit.IStoredSettings;
 import com.gitblit.Keys;
 import com.gitblit.models.FilestoreModel;
 import com.gitblit.models.FilestoreModel.Status;
 import com.gitblit.models.RepositoryModel;
 import com.gitblit.models.UserModel;
 import com.gitblit.utils.ArrayUtils;
 import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter;
 import com.google.gson.ExclusionStrategy;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.reflect.TypeToken;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;

 /**
  * FilestoreManager handles files uploaded via:
  * 	+ git-lfs
  *  + ticket attachment (TBD)
  *
  * Files are stored using their SHA256 hash (as per git-lfs)
  * If the same file is uploaded through different repositories no additional space is used
  * Access is controlled through the current repository permissions.
  *
  * TODO: Identify what and how the actual BLOBs should work with federation
  *
  * @author Paul Martin
  *
  */
 @Singleton
 public class FilestoreManager implements IFilestoreManager {

 	private final Logger logger = LoggerFactory.getLogger(getClass());

 	private final IRuntimeManager runtimeManager;

 	private final IRepositoryManager repositoryManager;

 	private final IStoredSettings settings;

 	public static final int UNDEFINED_SIZE = -1;

 	private static final String METAFILE = "filestore.json";

 	private static final String METAFILE_TMP = "filestore.json.tmp";

 	protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType();

 	private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>();


 	@Inject
 	FilestoreManager(
 			IRuntimeManager runtimeManager,
 			IRepositoryManager repositoryManager) {
 		this.runtimeManager = runtimeManager;
 		this.repositoryManager = repositoryManager;
 		this.settings = runtimeManager.getSettings();
 	}

 	@Override
 	public IManager start() {

 		// Try to load any existing metadata
 		File dir = getStorageFolder();
 		dir.mkdirs();
 		File metadata = new File(dir, METAFILE);

 		if (metadata.exists()) {
 			Collection<FilestoreModel> items = null;

 			Gson gson = gson();
 			try (FileReader file = new FileReader(metadata)) {
 				items = gson.fromJson(file, METAFILE_TYPE);
 				file.close();

 			} catch (IOException e) {
 				e.printStackTrace();
 			}

 			for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) {
 			    FilestoreModel model = itr.next();
 			    fileCache.put(model.oid, model);
 			}

 			logger.info("Loaded {} items from filestore metadata file", fileCache.size());
 		}
 		else
 		{
 			logger.info("No filestore metadata file found");
 		}

 		return this;
 	}

 	@Override
 	public IManager stop() {
 		return this;
 	}


 	@Override
 	public boolean isValidOid(String oid) {
 		//NOTE: Assuming SHA256 support only as per git-lfs
 		return Pattern.matches("[a-fA-F0-9]{64}", oid);
 	}

 	@Override
 	public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) {

 		//Handle access control
 		if (!user.canPush(repo)) {
 			if (user == UserModel.ANONYMOUS) {
 				return Status.AuthenticationRequired;
 			} else {
 				return Status.Error_Unauthorized;
 			}
 		}

 		//Handle object details
 		if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; }

 		if (fileCache.containsKey(oid)) {
 			FilestoreModel item = fileCache.get(oid);

 			if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) {
 				return Status.Error_Size_Mismatch;
 			}

 			item.addRepository(repo.name);

 			if (item.isInErrorState()) {
 				item.reset(user, size);
 			}
 		} else {

 			if (size  < 0) {return Status.Error_Invalid_Size; }
 			if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; }

 			FilestoreModel model = new FilestoreModel(oid, size, user, repo.name);
 			fileCache.put(oid, model);
 			saveFilestoreModel(model);
 		}

 		return fileCache.get(oid).getStatus();
 	}

 	@Override
 	public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) {

 		//Access control and object logic
 		Status state = addObject(oid, size, user, repo);

 		if (state != Status.Upload_Pending) {
 			return state;
 		}

 		FilestoreModel model = fileCache.get(oid);

 		if (!model.actionUpload(user)) {
 			return Status.Upload_In_Progress;
 		} else {
 			long actualSize = 0;
 			File file = getStoragePath(oid);

 			try {
 				file.getParentFile().mkdirs();
 				file.createNewFile();

 				try (FileOutputStream streamOut = new FileOutputStream(file)) {

 					actualSize = IOUtils.copyLarge(streamIn, streamOut);

 					streamOut.flush();
 					streamOut.close();

 					if (model.getSize() != actualSize) {
 						model.setStatus(Status.Error_Size_Mismatch, user);

 						logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}",
 								oid, model.getSize(), actualSize));
 					} else {
 						String actualOid = "";

 						try (FileInputStream fileForHash = new FileInputStream(file)) {
 							actualOid = DigestUtils.sha256Hex(fileForHash);
 							fileForHash.close();
 						}

 						if (oid.equalsIgnoreCase(actualOid)) {
 							model.setStatus(Status.Available, user);
 						} else {
 							model.setStatus(Status.Error_Hash_Mismatch, user);

 							logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid));
 						}
 					}
 				}
 			} catch (Exception e) {

 				model.setStatus(Status.Error_Unknown, user);
 				logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e);
 			} finally {
 				saveFilestoreModel(model);
 			}

 			if (model.isInErrorState()) {
 				file.delete();
 				model.removeRepository(repo.name);
 			}
 		}

 		return model.getStatus();
 	}

 	private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) {

 		//Access Control
 		if (!user.canView(repo)) {
 			if (user == UserModel.ANONYMOUS) {
 				return Status.AuthenticationRequired;
 			} else {
 				return Status.Error_Unauthorized;
 			}
 		}

 		//Object Logic
 		if (!isValidOid(oid)) {
 			return Status.Error_Invalid_Oid;
 		}

 		if (!fileCache.containsKey(oid)) {
 			return Status.Unavailable;
 		}

 		FilestoreModel item = fileCache.get(oid);

 		if (item.getStatus() == Status.Available) {
 			return Status.Available;
 		}

 		return Status.Unavailable;
 	}

 	@Override
 	public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) {

 		if (canGetObject(oid, user, repo) == Status.Available) {
 			return fileCache.get(oid);
 		}

 		return null;
 	}

 	@Override
 	public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) {

 		//Access control and object logic
 		Status status = canGetObject(oid, user, repo);

 		if (status != Status.Available) {
 			return status;
 		}

 		FilestoreModel item = fileCache.get(oid);

 		if (streamOut != null) {
 			try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) {

 				IOUtils.copyLarge(streamIn, streamOut);

 				streamOut.flush();
 				streamIn.close();
 			} catch (EOFException e) {
 				logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e);
 				return Status.Error_Unexpected_Stream_End;
 			} catch (Exception e) {
 				logger.error(MessageFormat.format("Failed to download blob {0}", oid), e);
 				return Status.Error_Unknown;
 			}
 		}

 		return item.getStatus();
 	}

 	@Override
 	public List<FilestoreModel> getAllObjects(UserModel user) {

 		final List<RepositoryModel> viewableRepositories = repositoryManager.getRepositoryModels(user);
 		List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size());

 		for (RepositoryModel repository : viewableRepositories) {
 			viewableRepositoryNames.add(repository.name);
 		}

 		if (viewableRepositoryNames.size() == 0) {
 			return null;
 		}

 		final Collection<FilestoreModel> allFiles = fileCache.values();
 		List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size());

 		for (FilestoreModel file : allFiles) {
 			if (file.isInRepositoryList(viewableRepositoryNames)) {
 				userViewableFiles.add(file);
 			}
 		}

 		return userViewableFiles;
 	}

 	@Override
 	public File getStorageFolder() {
 		return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs");
 	}

 	@Override
 	public File getStoragePath(String oid) {
 		 return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2)));
 	}

 	@Override
 	public long getMaxUploadSize() {
 		return settings.getLong(Keys.filestore.maxUploadSize, -1);
 	}

 	@Override
 	public long getFilestoreUsedByteCount() {
 		Iterator<FilestoreModel> iterator = fileCache.values().iterator();
 		long total = 0;

 		while (iterator.hasNext()) {

 			FilestoreModel item = iterator.next();
 			if (item.getStatus() == Status.Available) {
 				total += item.getSize();
 			}
 		}

 		return total;
 	}

 	@Override
 	public long getFilestoreAvailableByteCount() {

 		try {
 			return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace();
 		} catch (IOException e) {
 			logger.error(MessageFormat.format("Failed to retrieve available space in Filestore {0}", e));
 		}

 		return UNDEFINED_SIZE;
 	};

 	private synchronized void saveFilestoreModel(FilestoreModel model) {

 		File metaFile = new File(getStorageFolder(), METAFILE);
 		File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP);
 		boolean isNewFile = false;

 		try {
 			if (!metaFile.exists()) {
 				metaFile.getParentFile().mkdirs();
 				metaFile.createNewFile();
 				isNewFile = true;
 			}
 			FileUtils.copyFile(metaFile, metaFileTmp);

 		} catch (IOException e) {
 			logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
 		}

 		try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) {

 			if (isNewFile) {
 				fs.writeBytes("[");
 			} else {
 				fs.seek(fs.length() - 1);
 				fs.writeBytes(",");
 			}

 			fs.writeBytes(gson().toJson(model));
 			fs.writeBytes("]");

 			fs.close();

 		} catch (IOException e) {
 			logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e);
 		}

 		try {
 			if (metaFileTmp.exists()) {
 				FileUtils.copyFile(metaFileTmp, metaFile);

 				metaFileTmp.delete();
 			} else {
 				logger.error("Writing filestore model to file {0}", METAFILE);
 			}
 		}
 		catch (IOException e) {
 			logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
 		}
 	}

 	/*
 	 * Intended for testing purposes only
 	 */
 	@Override
 	public void clearFilestoreCache() {
 		fileCache.clear();
 	}

 	private static Gson gson(ExclusionStrategy... strategies) {
 		GsonBuilder builder = new GsonBuilder();
 		builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter());
 		if (!ArrayUtils.isEmpty(strategies)) {
 			builder.setExclusionStrategies(strategies);
 		}
 		return builder.create();
 	}

 }
	/*
	* Copyright 2015 gitblit.com.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package com.gitblit.manager;

	import java.io.EOFException;
	import java.io.File;
	import java.io.FileInputStream;
	import java.io.FileOutputStream;
	import java.io.FileReader;
	import java.io.IOException;
	import java.io.InputStream;
	import java.io.OutputStream;
	import java.io.RandomAccessFile;
	import java.lang.reflect.Type;
	import java.nio.file.Files;
	import java.text.MessageFormat;
	import java.util.ArrayList;
	import java.util.Collection;
	import java.util.Date;
	import java.util.Iterator;
	import java.util.List;
	import java.util.Map;
	import java.util.concurrent.ConcurrentHashMap;
	import java.util.regex.Pattern;

	import org.apache.commons.codec.digest.DigestUtils;
	import org.apache.commons.io.FileUtils;
	import org.apache.commons.io.IOUtils;
	import org.slf4j.Logger;
	import org.slf4j.LoggerFactory;

	import com.gitblit.IStoredSettings;
	import com.gitblit.Keys;
	import com.gitblit.models.FilestoreModel;
	import com.gitblit.models.FilestoreModel.Status;
	import com.gitblit.models.RepositoryModel;
	import com.gitblit.models.UserModel;
	import com.gitblit.utils.ArrayUtils;
	import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter;
	import com.google.gson.ExclusionStrategy;
	import com.google.gson.Gson;
	import com.google.gson.GsonBuilder;
	import com.google.gson.reflect.TypeToken;
	import com.google.inject.Inject;
	import com.google.inject.Singleton;

	/**
	* FilestoreManager handles files uploaded via:
	* + git-lfs
	* + ticket attachment (TBD)
	*
	* Files are stored using their SHA256 hash (as per git-lfs)
	* If the same file is uploaded through different repositories no additional space is used
	* Access is controlled through the current repository permissions.
	*
	* TODO: Identify what and how the actual BLOBs should work with federation
	*
	* @author Paul Martin
	*
	*/
	@Singleton
	public class FilestoreManager implements IFilestoreManager {

	private final Logger logger = LoggerFactory.getLogger(getClass());

	private final IRuntimeManager runtimeManager;

	private final IRepositoryManager repositoryManager;

	private final IStoredSettings settings;

	public static final int UNDEFINED_SIZE = -1;

	private static final String METAFILE = "filestore.json";

	private static final String METAFILE_TMP = "filestore.json.tmp";

	protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType();

	private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>();


	@Inject
	FilestoreManager(
	IRuntimeManager runtimeManager,
	IRepositoryManager repositoryManager) {
	this.runtimeManager = runtimeManager;
	this.repositoryManager = repositoryManager;
	this.settings = runtimeManager.getSettings();
	}

	@Override
	public IManager start() {

	// Try to load any existing metadata
	File dir = getStorageFolder();
	dir.mkdirs();
	File metadata = new File(dir, METAFILE);

	if (metadata.exists()) {
	Collection<FilestoreModel> items = null;

	Gson gson = gson();
	try (FileReader file = new FileReader(metadata)) {
	items = gson.fromJson(file, METAFILE_TYPE);
	file.close();

	} catch (IOException e) {
	e.printStackTrace();
	}

	for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) {
	FilestoreModel model = itr.next();
	fileCache.put(model.oid, model);
	}

	logger.info("Loaded {} items from filestore metadata file", fileCache.size());
	}
	else
	{
	logger.info("No filestore metadata file found");
	}

	return this;
	}

	@Override
	public IManager stop() {
	return this;
	}


	@Override
	public boolean isValidOid(String oid) {
	//NOTE: Assuming SHA256 support only as per git-lfs
	return Pattern.matches("[a-fA-F0-9]{64}", oid);
	}

	@Override
	public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) {

	//Handle access control
	if (!user.canPush(repo)) {
	if (user == UserModel.ANONYMOUS) {
	return Status.AuthenticationRequired;
	} else {
	return Status.Error_Unauthorized;
	}
	}

	//Handle object details
	if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; }

	if (fileCache.containsKey(oid)) {
	FilestoreModel item = fileCache.get(oid);

	if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) {
	return Status.Error_Size_Mismatch;
	}

	item.addRepository(repo.name);

	if (item.isInErrorState()) {
	item.reset(user, size);
	}
	} else {

	if (size < 0) {return Status.Error_Invalid_Size; }
	if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; }

	FilestoreModel model = new FilestoreModel(oid, size, user, repo.name);
	fileCache.put(oid, model);
	saveFilestoreModel(model);
	}

	return fileCache.get(oid).getStatus();
	}

	@Override
	public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) {

	//Access control and object logic
	Status state = addObject(oid, size, user, repo);

	if (state != Status.Upload_Pending) {
	return state;
	}

	FilestoreModel model = fileCache.get(oid);

	if (!model.actionUpload(user)) {
	return Status.Upload_In_Progress;
	} else {
	long actualSize = 0;
	File file = getStoragePath(oid);

	try {
	file.getParentFile().mkdirs();
	file.createNewFile();

	try (FileOutputStream streamOut = new FileOutputStream(file)) {

	actualSize = IOUtils.copyLarge(streamIn, streamOut);

	streamOut.flush();
	streamOut.close();

	if (model.getSize() != actualSize) {
	model.setStatus(Status.Error_Size_Mismatch, user);

	logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}",
	oid, model.getSize(), actualSize));
	} else {
	String actualOid = "";

	try (FileInputStream fileForHash = new FileInputStream(file)) {
	actualOid = DigestUtils.sha256Hex(fileForHash);
	fileForHash.close();
	}

	if (oid.equalsIgnoreCase(actualOid)) {
	model.setStatus(Status.Available, user);
	} else {
	model.setStatus(Status.Error_Hash_Mismatch, user);

	logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid));
	}
	}
	}
	} catch (Exception e) {

	model.setStatus(Status.Error_Unknown, user);
	logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e);
	} finally {
	saveFilestoreModel(model);
	}

	if (model.isInErrorState()) {
	file.delete();
	model.removeRepository(repo.name);
	}
	}

	return model.getStatus();
	}

	private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) {

	//Access Control
	if (!user.canView(repo)) {
	if (user == UserModel.ANONYMOUS) {
	return Status.AuthenticationRequired;
	} else {
	return Status.Error_Unauthorized;
	}
	}

	//Object Logic
	if (!isValidOid(oid)) {
	return Status.Error_Invalid_Oid;
	}

	if (!fileCache.containsKey(oid)) {
	return Status.Unavailable;
	}

	FilestoreModel item = fileCache.get(oid);

	if (item.getStatus() == Status.Available) {
	return Status.Available;
	}

	return Status.Unavailable;
	}

	@Override
	public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) {

	if (canGetObject(oid, user, repo) == Status.Available) {
	return fileCache.get(oid);
	}

	return null;
	}

	@Override
	public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) {

	//Access control and object logic
	Status status = canGetObject(oid, user, repo);

	if (status != Status.Available) {
	return status;
	}

	FilestoreModel item = fileCache.get(oid);

	if (streamOut != null) {
	try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) {

	IOUtils.copyLarge(streamIn, streamOut);

	streamOut.flush();
	streamIn.close();
	} catch (EOFException e) {
	logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e);
	return Status.Error_Unexpected_Stream_End;
	} catch (Exception e) {
	logger.error(MessageFormat.format("Failed to download blob {0}", oid), e);
	return Status.Error_Unknown;
	}
	}

	return item.getStatus();
	}

	@Override
	public List<FilestoreModel> getAllObjects(UserModel user) {

	final List<RepositoryModel> viewableRepositories = repositoryManager.getRepositoryModels(user);
	List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size());

	for (RepositoryModel repository : viewableRepositories) {
	viewableRepositoryNames.add(repository.name);
	}

	if (viewableRepositoryNames.size() == 0) {
	return null;
	}

	final Collection<FilestoreModel> allFiles = fileCache.values();
	List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size());

	for (FilestoreModel file : allFiles) {
	if (file.isInRepositoryList(viewableRepositoryNames)) {
	userViewableFiles.add(file);
	}
	}

	return userViewableFiles;
	}

	@Override
	public File getStorageFolder() {
	return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs");
	}

	@Override
	public File getStoragePath(String oid) {
	return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2)));
	}

	@Override
	public long getMaxUploadSize() {
	return settings.getLong(Keys.filestore.maxUploadSize, -1);
	}

	@Override
	public long getFilestoreUsedByteCount() {
	Iterator<FilestoreModel> iterator = fileCache.values().iterator();
	long total = 0;

	while (iterator.hasNext()) {

	FilestoreModel item = iterator.next();
	if (item.getStatus() == Status.Available) {
	total += item.getSize();
	}
	}

	return total;
	}

	@Override
	public long getFilestoreAvailableByteCount() {

	try {
	return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace();
	} catch (IOException e) {
	logger.error(MessageFormat.format("Failed to retrieve available space in Filestore {0}", e));
	}

	return UNDEFINED_SIZE;
	};

	private synchronized void saveFilestoreModel(FilestoreModel model) {

	File metaFile = new File(getStorageFolder(), METAFILE);
	File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP);
	boolean isNewFile = false;

	try {
	if (!metaFile.exists()) {
	metaFile.getParentFile().mkdirs();
	metaFile.createNewFile();
	isNewFile = true;
	}
	FileUtils.copyFile(metaFile, metaFileTmp);

	} catch (IOException e) {
	logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
	}

	try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) {

	if (isNewFile) {
	fs.writeBytes("[");
	} else {
	fs.seek(fs.length() - 1);
	fs.writeBytes(",");
	}

	fs.writeBytes(gson().toJson(model));
	fs.writeBytes("]");

	fs.close();

	} catch (IOException e) {
	logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e);
	}

	try {
	if (metaFileTmp.exists()) {
	FileUtils.copyFile(metaFileTmp, metaFile);

	metaFileTmp.delete();
	} else {
	logger.error("Writing filestore model to file {0}", METAFILE);
	}
	}
	catch (IOException e) {
	logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
	}
	}

	/*
	* Intended for testing purposes only
	*/
	@Override
	public void clearFilestoreCache() {
	fileCache.clear();
	}

	private static Gson gson(ExclusionStrategy... strategies) {
	GsonBuilder builder = new GsonBuilder();
	builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter());
	if (!ArrayUtils.isEmpty(strategies)) {
	builder.setExclusionStrategies(strategies);
	}
	return builder.create();
	}

	}