diff --git a/bits_helpers/git.py b/bits_helpers/git.py index 8bb0d2d8..324483d6 100644 --- a/bits_helpers/git.py +++ b/bits_helpers/git.py @@ -70,6 +70,9 @@ def cloneSourceCmd(self, source, destination, referenceRepo, usePartialClone): def checkoutCmd(self, tag): return ["checkout", "-f", tag] + def cleanCmd(self): + return ["clean", "-fdxq"] + def fetchCmd(self, remote, *refs): return ["fetch", "-f", "--prune"] + clone_speedup_options() + [remote, *refs] diff --git a/bits_helpers/scm.py b/bits_helpers/scm.py index bea737b8..83946692 100644 --- a/bits_helpers/scm.py +++ b/bits_helpers/scm.py @@ -18,6 +18,8 @@ def exec(self, *args, **kwargs): raise NotImplementedError def checkoutCmd(self, tag): raise NotImplementedError + def cleanCmd(self): + raise NotImplementedError def fetchCmd(self, remote, *refs): raise NotImplementedError def cloneReferenceCmd(self, spec, referenceRepo, usePartialClone): diff --git a/bits_helpers/sl.py b/bits_helpers/sl.py index 51adc43f..66af3611 100644 --- a/bits_helpers/sl.py +++ b/bits_helpers/sl.py @@ -52,6 +52,9 @@ def parseRefs(self, output): def listRefsCmd(self, repository): return ["bookmark", "--list", "--remote", "-R", repository] + def cleanCmd(self): + return ["clean", "--ignored"] + def diffCmd(self, directory): return "cd %s && sl diff && sl status" % directory diff --git a/bits_helpers/workarea.py b/bits_helpers/workarea.py index d5495347..64ecdd70 100644 --- a/bits_helpers/workarea.py +++ b/bits_helpers/workarea.py @@ -825,7 +825,9 @@ def scm_exec(command, directory=".", check=True): _source_checksums = spec.get("source_checksums") or {} _patch_checksums = spec.get("patch_checksums") or {} - if spec.get("patches"): + def _stage_patches(): + if not spec.get("patches"): + return os.makedirs(source_dir, exist_ok=True) for patch_entry in spec["patches"]: patch_name, inline_checksum = parse_entry(patch_entry) @@ -833,7 +835,9 @@ def scm_exec(command, directory=".", check=True): dst = os.path.join(source_dir, patch_name) shutil.copyfile(os.path.join(spec["pkgdir"], 'patches', patch_name), dst) check_file_checksum(dst, patch_name, patch_checksum, enforce_mode) + if spec.get("sources"): + _stage_patches() # Resolve arch-conditional / bash-evaluated source entries before # downloading. ``architecture`` is passed in by the caller (build.py # knows raw_architecture); fall back to the env var for backwards @@ -897,6 +901,7 @@ def _download_one(s): # present in the JSON but an empty string. package_info("%s: No sources to fetch", package_label(spec)) os.makedirs(source_dir, exist_ok=True) + _stage_patches() elif spec["is_devel_pkg"]: report_source("local directory " + spec["source"]) shutil.rmtree(source_dir, ignore_errors=True) @@ -917,6 +922,9 @@ def _download_one(s): scm_exec(scm.fetchCmd(spec["source"], tag_ref), source_dir) scm_exec(scm.checkoutCmd(spec["tag"]), source_dir) _verify_commit_pin(scm, spec, source_dir, enforce_mode) + # clean even without patches, otherwise files added by a dropped patch stay around + scm_exec(scm.cleanCmd(), source_dir) + _stage_patches() _apply_patches(spec, source_dir) else: # source_dir either doesn't exist yet, OR exists but is not a valid checkout @@ -948,4 +956,5 @@ def _download_one(s): scm_exec(scm.setWriteUrlCmd(spec.get("write_repo", spec["source"])), source_dir) scm_exec(scm.checkoutCmd(spec["tag"]), source_dir) _verify_commit_pin(scm, spec, source_dir, enforce_mode) + _stage_patches() _apply_patches(spec, source_dir) diff --git a/tests/test_workarea.py b/tests/test_workarea.py index 4393bc49..a4ee8f2a 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -363,6 +363,112 @@ def test_no_sentinel_on_failure(self, mock_run, mock_die): ".bits_patched must not exist after a failed patch application") +@unittest.skipUnless(shutil.which("git") and shutil.which("patch"), "needs git and patch") +@patch("bits_helpers.workarea.debug", new=MagicMock()) +class GitSourcePatchTest(unittest.TestCase): + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.repo = os.path.join(self.tmp, "upstream") + self.work_dir = os.path.join(self.tmp, "sw") + self.pkgdir = os.path.join(self.tmp, "recipes") + os.makedirs(self.repo) + os.makedirs(os.path.join(self.pkgdir, "patches")) + with open(os.path.join(self.repo, "hello.txt"), "w") as fh: + fh.write("hello\n") + git = ["git", "-C", self.repo, "-c", "user.name=t", "-c", "user.email=t@t"] + subprocess.run(["git", "init", "-q", self.repo], check=True) + subprocess.run(git + ["add", "hello.txt"], check=True) + subprocess.run(git + ["commit", "-q", "-m", "init"], check=True) + subprocess.run(git + ["tag", "v1.0"], check=True) + with open(os.path.join(self.pkgdir, "patches", "hello.patch"), "w") as fh: + fh.write("--- a/hello.txt\n+++ b/hello.txt\n@@ -1 +1 @@\n-hello\n+hello patched\n") + with open(os.path.join(self.pkgdir, "patches", "new.patch"), "w") as fh: + fh.write("--- /dev/null\n+++ b/new.txt\n@@ -0,0 +1 @@\n+new\n") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _checkout(self, patches=("hello.patch",)): + from bits_helpers.workarea import checkout_sources + from bits_helpers.utilities import short_commit_hash + spec = OrderedDict(( + ("package", "mylib"), + ("version", "1.0"), + ("source", self.repo), + ("tag", "v1.0"), + ("commit_hash", "v1.0"), + ("scm", Git()), + ("is_devel_pkg", False), + ("pkgdir", self.pkgdir), + ("patches", list(patches)), + )) + checkout_sources(spec, self.work_dir, None, False) + return os.path.join(self.work_dir, "SOURCES", "mylib", "1.0", short_commit_hash(spec)) + + def _read(self, source_dir, name): + with open(os.path.join(source_dir, name)) as fh: + return fh.read() + + def test_fresh_clone_is_patched(self): + """First build: the patch is applied after the clone.""" + self.assertEqual(self._read(self._checkout(), "hello.txt"), "hello patched\n", + "patch was not applied after cloning") + + def test_existing_checkout_is_patched_again(self): + """Rebuild: the reused checkout is reset and patched again.""" + self._checkout() + self.assertEqual(self._read(self._checkout(), "hello.txt"), "hello patched\n", + "reused checkout was built without its patch") + + def test_a_file_a_patch_creates_stays_on_every_build(self): + """A file added by a patch is there on every build, not only the first.""" + for build in range(1, 4): + self.assertEqual(self._read(self._checkout(["new.patch"]), "new.txt"), "new\n", + "new.txt missing on build %d" % build) + + def test_a_file_from_a_dropped_patch_is_removed(self): + """Dropping a patch from the recipe also removes the file it added.""" + self._checkout(["hello.patch", "new.patch"]) + source_dir = self._checkout(["hello.patch"]) + self.assertFalse(os.path.exists(os.path.join(source_dir, "new.txt")), + "new.txt from the dropped patch is still there") + + def test_a_package_without_patches_gets_a_clean_tree(self): + """No patches: leftovers from the last build are still removed.""" + source_dir = self._checkout(()) + with open(os.path.join(source_dir, "leftover.txt"), "w") as fh: + fh.write("from the last build\n") + self._checkout(()) + self.assertFalse(os.path.exists(os.path.join(source_dir, "leftover.txt")), + "leftover.txt survived the rebuild") + self.assertEqual(self._read(source_dir, "hello.txt"), "hello\n", + "the source itself was changed") + + def test_an_ignored_file_is_removed(self): + """Files matched by .gitignore (e.g. generated ones) are removed too.""" + source_dir = self._checkout() + with open(os.path.join(source_dir, ".git", "info", "exclude"), "a") as fh: + fh.write("*.gen\n") + with open(os.path.join(source_dir, "configure.gen"), "w") as fh: + fh.write("generated\n") + self._checkout() + self.assertFalse(os.path.exists(os.path.join(source_dir, "configure.gen")), + "ignored file survived the rebuild (clean without -x?)") + + def test_a_nested_git_repository_is_kept(self): + """A nested repository, like a submodule, is left alone.""" + source_dir = self._checkout() + sub = os.path.join(source_dir, "sub") + subprocess.run(["git", "init", "-q", sub], check=True) + with open(os.path.join(sub, "file.txt"), "w") as fh: + fh.write("submodule\n") + self._checkout() + self.assertTrue(os.path.exists(os.path.join(sub, "file.txt")), + "nested repository was deleted (clean with -ff?)") + self.assertEqual(self._read(sub, "file.txt"), "submodule\n") + + class SourceCheckoutValidityTest(unittest.TestCase): """_is_valid_source_checkout: a git source dir with no .git is not usable."""