| /* |
| * Copyright 2015 gitblit.com. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package com.gitblit.manager; |
| |
| import java.io.EOFException; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileOutputStream; |
| import java.io.FileReader; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.OutputStream; |
| import java.io.RandomAccessFile; |
| import java.lang.reflect.Type; |
| import java.nio.file.Files; |
| import java.text.MessageFormat; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.Date; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.concurrent.ConcurrentHashMap; |
| import java.util.regex.Pattern; |
| |
| import org.apache.commons.codec.digest.DigestUtils; |
| import org.apache.commons.io.FileUtils; |
| import org.apache.commons.io.IOUtils; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import com.gitblit.IStoredSettings; |
| import com.gitblit.Keys; |
| import com.gitblit.models.FilestoreModel; |
| import com.gitblit.models.FilestoreModel.Status; |
| import com.gitblit.models.RepositoryModel; |
| import com.gitblit.models.UserModel; |
| import com.gitblit.utils.ArrayUtils; |
| import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter; |
| import com.google.gson.ExclusionStrategy; |
| import com.google.gson.Gson; |
| import com.google.gson.GsonBuilder; |
| import com.google.gson.reflect.TypeToken; |
| import com.google.inject.Inject; |
| import com.google.inject.Singleton; |
| |
| /** |
| * FilestoreManager handles files uploaded via: |
| * + git-lfs |
| * + ticket attachment (TBD) |
| * |
| * Files are stored using their SHA256 hash (as per git-lfs) |
| * If the same file is uploaded through different repositories no additional space is used |
| * Access is controlled through the current repository permissions. |
| * |
| * TODO: Identify what and how the actual BLOBs should work with federation |
| * |
| * @author Paul Martin |
| * |
| */ |
| @Singleton |
| public class FilestoreManager implements IFilestoreManager { |
| |
| private final Logger logger = LoggerFactory.getLogger(getClass()); |
| |
| private final IRuntimeManager runtimeManager; |
| |
| private final IRepositoryManager repositoryManager; |
| |
| private final IStoredSettings settings; |
| |
| public static final int UNDEFINED_SIZE = -1; |
| |
| private static final String METAFILE = "filestore.json"; |
| |
| private static final String METAFILE_TMP = "filestore.json.tmp"; |
| |
| protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType(); |
| |
| private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>(); |
| |
| |
| @Inject |
| FilestoreManager( |
| IRuntimeManager runtimeManager, |
| IRepositoryManager repositoryManager) { |
| this.runtimeManager = runtimeManager; |
| this.repositoryManager = repositoryManager; |
| this.settings = runtimeManager.getSettings(); |
| } |
| |
| @Override |
| public IManager start() { |
| |
| // Try to load any existing metadata |
| File dir = getStorageFolder(); |
| dir.mkdirs(); |
| File metadata = new File(dir, METAFILE); |
| |
| if (metadata.exists()) { |
| Collection<FilestoreModel> items = null; |
| |
| Gson gson = gson(); |
| try (FileReader file = new FileReader(metadata)) { |
| items = gson.fromJson(file, METAFILE_TYPE); |
| file.close(); |
| |
| } catch (IOException e) { |
| e.printStackTrace(); |
| } |
| |
| for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) { |
| FilestoreModel model = itr.next(); |
| fileCache.put(model.oid, model); |
| } |
| |
| logger.info("Loaded {} items from filestore metadata file", fileCache.size()); |
| } |
| else |
| { |
| logger.info("No filestore metadata file found"); |
| } |
| |
| return this; |
| } |
| |
| @Override |
| public IManager stop() { |
| return this; |
| } |
| |
| |
| @Override |
| public boolean isValidOid(String oid) { |
| //NOTE: Assuming SHA256 support only as per git-lfs |
| return Pattern.matches("[a-fA-F0-9]{64}", oid); |
| } |
| |
| @Override |
| public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) { |
| |
| //Handle access control |
| if (!user.canPush(repo)) { |
| if (user == UserModel.ANONYMOUS) { |
| return Status.AuthenticationRequired; |
| } else { |
| return Status.Error_Unauthorized; |
| } |
| } |
| |
| //Handle object details |
| if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; } |
| |
| if (fileCache.containsKey(oid)) { |
| FilestoreModel item = fileCache.get(oid); |
| |
| if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) { |
| return Status.Error_Size_Mismatch; |
| } |
| |
| item.addRepository(repo.name); |
| |
| if (item.isInErrorState()) { |
| item.reset(user, size); |
| } |
| } else { |
| |
| if (size < 0) {return Status.Error_Invalid_Size; } |
| if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; } |
| |
| FilestoreModel model = new FilestoreModel(oid, size, user, repo.name); |
| fileCache.put(oid, model); |
| saveFilestoreModel(model); |
| } |
| |
| return fileCache.get(oid).getStatus(); |
| } |
| |
| @Override |
| public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) { |
| |
| //Access control and object logic |
| Status state = addObject(oid, size, user, repo); |
| |
| if (state != Status.Upload_Pending) { |
| return state; |
| } |
| |
| FilestoreModel model = fileCache.get(oid); |
| |
| if (!model.actionUpload(user)) { |
| return Status.Upload_In_Progress; |
| } else { |
| long actualSize = 0; |
| File file = getStoragePath(oid); |
| |
| try { |
| file.getParentFile().mkdirs(); |
| file.createNewFile(); |
| |
| try (FileOutputStream streamOut = new FileOutputStream(file)) { |
| |
| actualSize = IOUtils.copyLarge(streamIn, streamOut); |
| |
| streamOut.flush(); |
| streamOut.close(); |
| |
| if (model.getSize() != actualSize) { |
| model.setStatus(Status.Error_Size_Mismatch, user); |
| |
| logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}", |
| oid, model.getSize(), actualSize)); |
| } else { |
| String actualOid = ""; |
| |
| try (FileInputStream fileForHash = new FileInputStream(file)) { |
| actualOid = DigestUtils.sha256Hex(fileForHash); |
| fileForHash.close(); |
| } |
| |
| if (oid.equalsIgnoreCase(actualOid)) { |
| model.setStatus(Status.Available, user); |
| } else { |
| model.setStatus(Status.Error_Hash_Mismatch, user); |
| |
| logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid)); |
| } |
| } |
| } |
| } catch (Exception e) { |
| |
| model.setStatus(Status.Error_Unknown, user); |
| logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e); |
| } finally { |
| saveFilestoreModel(model); |
| } |
| |
| if (model.isInErrorState()) { |
| file.delete(); |
| model.removeRepository(repo.name); |
| } |
| } |
| |
| return model.getStatus(); |
| } |
| |
| private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) { |
| |
| //Access Control |
| if (!user.canView(repo)) { |
| if (user == UserModel.ANONYMOUS) { |
| return Status.AuthenticationRequired; |
| } else { |
| return Status.Error_Unauthorized; |
| } |
| } |
| |
| //Object Logic |
| if (!isValidOid(oid)) { |
| return Status.Error_Invalid_Oid; |
| } |
| |
| if (!fileCache.containsKey(oid)) { |
| return Status.Unavailable; |
| } |
| |
| FilestoreModel item = fileCache.get(oid); |
| |
| if (item.getStatus() == Status.Available) { |
| return Status.Available; |
| } |
| |
| return Status.Unavailable; |
| } |
| |
| @Override |
| public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) { |
| |
| if (canGetObject(oid, user, repo) == Status.Available) { |
| return fileCache.get(oid); |
| } |
| |
| return null; |
| } |
| |
| @Override |
| public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) { |
| |
| //Access control and object logic |
| Status status = canGetObject(oid, user, repo); |
| |
| if (status != Status.Available) { |
| return status; |
| } |
| |
| FilestoreModel item = fileCache.get(oid); |
| |
| if (streamOut != null) { |
| try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) { |
| |
| IOUtils.copyLarge(streamIn, streamOut); |
| |
| streamOut.flush(); |
| streamIn.close(); |
| } catch (EOFException e) { |
| logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e); |
| return Status.Error_Unexpected_Stream_End; |
| } catch (Exception e) { |
| logger.error(MessageFormat.format("Failed to download blob {0}", oid), e); |
| return Status.Error_Unknown; |
| } |
| } |
| |
| return item.getStatus(); |
| } |
| |
| @Override |
| public List<FilestoreModel> getAllObjects(UserModel user) { |
| |
| final List<RepositoryModel> viewableRepositories = repositoryManager.getRepositoryModels(user); |
| List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size()); |
| |
| for (RepositoryModel repository : viewableRepositories) { |
| viewableRepositoryNames.add(repository.name); |
| } |
| |
| if (viewableRepositoryNames.size() == 0) { |
| return null; |
| } |
| |
| final Collection<FilestoreModel> allFiles = fileCache.values(); |
| List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size()); |
| |
| for (FilestoreModel file : allFiles) { |
| if (file.isInRepositoryList(viewableRepositoryNames)) { |
| userViewableFiles.add(file); |
| } |
| } |
| |
| return userViewableFiles; |
| } |
| |
| @Override |
| public File getStorageFolder() { |
| return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs"); |
| } |
| |
| @Override |
| public File getStoragePath(String oid) { |
| return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2))); |
| } |
| |
| @Override |
| public long getMaxUploadSize() { |
| return settings.getLong(Keys.filestore.maxUploadSize, -1); |
| } |
| |
| @Override |
| public long getFilestoreUsedByteCount() { |
| Iterator<FilestoreModel> iterator = fileCache.values().iterator(); |
| long total = 0; |
| |
| while (iterator.hasNext()) { |
| |
| FilestoreModel item = iterator.next(); |
| if (item.getStatus() == Status.Available) { |
| total += item.getSize(); |
| } |
| } |
| |
| return total; |
| } |
| |
| @Override |
| public long getFilestoreAvailableByteCount() { |
| |
| try { |
| return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace(); |
| } catch (IOException e) { |
| logger.error(MessageFormat.format("Failed to retrive available space in Filestore {0}", e)); |
| } |
| |
| return UNDEFINED_SIZE; |
| }; |
| |
| private synchronized void saveFilestoreModel(FilestoreModel model) { |
| |
| File metaFile = new File(getStorageFolder(), METAFILE); |
| File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP); |
| boolean isNewFile = false; |
| |
| try { |
| if (!metaFile.exists()) { |
| metaFile.getParentFile().mkdirs(); |
| metaFile.createNewFile(); |
| isNewFile = true; |
| } |
| FileUtils.copyFile(metaFile, metaFileTmp); |
| |
| } catch (IOException e) { |
| logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); |
| } |
| |
| try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) { |
| |
| if (isNewFile) { |
| fs.writeBytes("["); |
| } else { |
| fs.seek(fs.length() - 1); |
| fs.writeBytes(","); |
| } |
| |
| fs.writeBytes(gson().toJson(model)); |
| fs.writeBytes("]"); |
| |
| fs.close(); |
| |
| } catch (IOException e) { |
| logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e); |
| } |
| |
| try { |
| if (metaFileTmp.exists()) { |
| FileUtils.copyFile(metaFileTmp, metaFile); |
| |
| metaFileTmp.delete(); |
| } else { |
| logger.error("Writing filestore model to file {0}", METAFILE); |
| } |
| } |
| catch (IOException e) { |
| logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); |
| } |
| } |
| |
| /* |
| * Intended for testing purposes only |
| */ |
| @Override |
| public void clearFilestoreCache() { |
| fileCache.clear(); |
| } |
| |
| private static Gson gson(ExclusionStrategy... strategies) { |
| GsonBuilder builder = new GsonBuilder(); |
| builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter()); |
| if (!ArrayUtils.isEmpty(strategies)) { |
| builder.setExclusionStrategies(strategies); |
| } |
| return builder.create(); |
| } |
| |
| } |