-
Notifications
You must be signed in to change notification settings - Fork 545
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
WIP: Support converting repos to Git LFS
Git LFS allows uses to commit new files to the LFS store, but replacing _old_ files requires rewriting history, which is something the BFG is pretty good at. This rough cut allows replacing blobs with pointer files throughout repo history. Some caveats with this initial implementation: * the BFG cleans concurrently, files may unnecessarily be hashed more than once * the working directory isn't updated * specifying `-fi *.png` should be unnecessary, should use gitattributes * need for `--no-blob-protection` is a hangover from normal BFG behaviour Example invocation: ``` $ git clone https://github.com/guardian/membership-frontend.git $ cd membership-frontend $ java -jar bfg.jar --convert-to-git-lfs -fi *.png --no-blob-protection ... $ ls .git/lfs/objects/ | head -2 0145f7c304ef33a43cc946e0a57b2213d24dcaf8462f3d3b332407a8b258369c 07010d5ddea536da56ebdbbb28386921c94abd476046a245b35cd47e8eb6e426 $ git reset --hard $ cat frontend/assets/images/favicons/152x152.png version https://git-lfs.github.com/spec/v1 oid sha256:0145f7c304ef33a43cc946e0a57b2213d24dcaf8462f3d3b332407a8b258369c size 1935 $ ``` https://git-lfs.github.com/ https://github.com/github/git-lfs/blob/5eb9bb01/docs/spec.md#the-pointer
- Loading branch information
Showing
4 changed files
with
117 additions
and
3 deletions.
There are no files selected for viewing
88 changes: 88 additions & 0 deletions
88
bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| /* | ||
| * Copyright (c) 2015 Roberto Tyley | ||
| * | ||
| * This file is part of 'BFG Repo-Cleaner' - a tool for removing large | ||
| * or troublesome blobs from Git repositories. | ||
| * | ||
| * BFG Repo-Cleaner is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * BFG Repo-Cleaner is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see http://www.gnu.org/licenses/ . | ||
| */ | ||
|
|
||
| package com.madgag.git.bfg.cleaner | ||
|
|
||
| import java.nio.charset.Charset | ||
| import java.security.{DigestInputStream, MessageDigest} | ||
|
|
||
| import com.google.common.io.ByteStreams | ||
| import com.madgag.git.ThreadLocalObjectDatabaseResources | ||
| import com.madgag.git.bfg.model.{FileName, TreeBlobEntry} | ||
| import org.apache.commons.codec.binary.Hex.encodeHexString | ||
| import org.eclipse.jgit.lib.Constants.OBJ_BLOB | ||
| import org.eclipse.jgit.lib.ObjectLoader | ||
|
|
||
| import scala.util.Try | ||
| import scalax.file.Path | ||
| import scalax.file.Path.createTempFile | ||
| import scalax.io.Resource | ||
|
|
||
| trait LfsBlobConverter extends TreeBlobModifier { | ||
|
|
||
| val threadLocalObjectDBResources: ThreadLocalObjectDatabaseResources | ||
|
|
||
| val lfsSuitableFiles: (FileName => Boolean) | ||
|
|
||
| val charset = Charset.forName("UTF-8") | ||
|
|
||
| val lfsObjectsDir: Path | ||
|
|
||
| override def fix(entry: TreeBlobEntry) = { | ||
| val oid = (for { | ||
| _ <- Some(entry.filename) filter lfsSuitableFiles | ||
| loader = threadLocalObjectDBResources.reader().open(entry.objectId) | ||
| (shaHex, lfsPath) <- buildLfsFileFrom(loader) | ||
| } yield { | ||
| val pointer = | ||
| s"""|version https://git-lfs.github.com/spec/v1 | ||
| |oid sha256:$shaHex | ||
| |size ${loader.getSize} | ||
| |""".stripMargin | ||
|
|
||
| threadLocalObjectDBResources.inserter().insert(OBJ_BLOB, pointer.getBytes(charset)) | ||
| }).getOrElse(entry.objectId) | ||
|
|
||
| (entry.mode, oid) | ||
| } | ||
|
|
||
| def buildLfsFileFrom(loader: ObjectLoader): Option[(String, Path)] = { | ||
| val tmpFile = createTempFile() | ||
|
|
||
| val digest = MessageDigest.getInstance("SHA-256") | ||
|
|
||
| for { | ||
| inStream <- Resource.fromInputStream(new DigestInputStream(loader.openStream(), digest)) | ||
| outStream <- tmpFile.outputStream() | ||
| } ByteStreams.copy(inStream, outStream) | ||
|
|
||
| val shaHex = encodeHexString(digest.digest()) | ||
|
|
||
| val lfsPath = lfsObjectsDir / shaHex | ||
|
|
||
| val ensureLfsFile = Try(if (!lfsPath.exists) tmpFile moveTo lfsPath).recover { | ||
| case _ => lfsPath.size.contains(loader.getSize) | ||
| } | ||
|
|
||
| Try(tmpFile.delete(force = true)) | ||
|
|
||
| for (_ <- ensureLfsFile.toOption) yield shaHex -> lfsPath | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters