From 2db383df4c72e26599bd37fe953a12250a1206e0 Mon Sep 17 00:00:00 2001 From: Maria Folina <139060140+m-fol@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:01:49 +0200 Subject: [PATCH 1/7] workarea: copy patches after cloning a git source --- bits_helpers/workarea.py | 9 +++++++- tests/test_workarea.py | 47 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/bits_helpers/workarea.py b/bits_helpers/workarea.py index d5495347..9488d86b 100644 --- a/bits_helpers/workarea.py +++ b/bits_helpers/workarea.py @@ -825,7 +825,9 @@ def scm_exec(command, directory=".", check=True): _source_checksums = spec.get("source_checksums") or {} _patch_checksums = spec.get("patch_checksums") or {} - if spec.get("patches"): + def _stage_patches(): + if not spec.get("patches"): + return os.makedirs(source_dir, exist_ok=True) for patch_entry in spec["patches"]: patch_name, inline_checksum = parse_entry(patch_entry) @@ -833,7 +835,9 @@ def scm_exec(command, directory=".", check=True): dst = os.path.join(source_dir, patch_name) shutil.copyfile(os.path.join(spec["pkgdir"], 'patches', patch_name), dst) check_file_checksum(dst, patch_name, patch_checksum, enforce_mode) + if spec.get("sources"): + _stage_patches() # Resolve arch-conditional / bash-evaluated source entries before # downloading. ``architecture`` is passed in by the caller (build.py # knows raw_architecture); fall back to the env var for backwards @@ -897,6 +901,7 @@ def _download_one(s): # present in the JSON but an empty string. package_info("%s: No sources to fetch", package_label(spec)) os.makedirs(source_dir, exist_ok=True) + _stage_patches() elif spec["is_devel_pkg"]: report_source("local directory " + spec["source"]) shutil.rmtree(source_dir, ignore_errors=True) @@ -917,6 +922,7 @@ def _download_one(s): scm_exec(scm.fetchCmd(spec["source"], tag_ref), source_dir) scm_exec(scm.checkoutCmd(spec["tag"]), source_dir) _verify_commit_pin(scm, spec, source_dir, enforce_mode) + _stage_patches() _apply_patches(spec, source_dir) else: # source_dir either doesn't exist yet, OR exists but is not a valid checkout @@ -948,4 +954,5 @@ def _download_one(s): scm_exec(scm.setWriteUrlCmd(spec.get("write_repo", spec["source"])), source_dir) scm_exec(scm.checkoutCmd(spec["tag"]), source_dir) _verify_commit_pin(scm, spec, source_dir, enforce_mode) + _stage_patches() _apply_patches(spec, source_dir) diff --git a/tests/test_workarea.py b/tests/test_workarea.py index 4393bc49..9e8148e6 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -363,6 +363,53 @@ def test_no_sentinel_on_failure(self, mock_run, mock_die): ".bits_patched must not exist after a failed patch application") +@unittest.skipUnless(shutil.which("git") and shutil.which("patch"), "needs git and patch") +@patch("bits_helpers.workarea.debug", new=MagicMock()) +class GitSourcePatchTest(unittest.TestCase): + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.repo = os.path.join(self.tmp, "upstream") + self.work_dir = os.path.join(self.tmp, "sw") + self.pkgdir = os.path.join(self.tmp, "recipes") + os.makedirs(self.repo) + os.makedirs(os.path.join(self.pkgdir, "patches")) + with open(os.path.join(self.repo, "hello.txt"), "w") as fh: + fh.write("hello\n") + git = ["git", "-C", self.repo, "-c", "user.name=t", "-c", "user.email=t@t"] + subprocess.run(["git", "init", "-q", self.repo], check=True) + subprocess.run(git + ["add", "hello.txt"], check=True) + subprocess.run(git + ["commit", "-q", "-m", "init"], check=True) + subprocess.run(git + ["tag", "v1.0"], check=True) + with open(os.path.join(self.pkgdir, "patches", "hello.patch"), "w") as fh: + fh.write("--- a/hello.txt\n+++ b/hello.txt\n@@ -1 +1 @@\n-hello\n+hello patched\n") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _checkout(self): + from bits_helpers.workarea import checkout_sources + from bits_helpers.utilities import short_commit_hash + spec = OrderedDict(( + ("package", "mylib"), + ("version", "1.0"), + ("source", self.repo), + ("tag", "v1.0"), + ("commit_hash", "v1.0"), + ("scm", Git()), + ("is_devel_pkg", False), + ("pkgdir", self.pkgdir), + ("patches", ["hello.patch"]), + )) + checkout_sources(spec, self.work_dir, None, False) + source_dir = os.path.join(self.work_dir, "SOURCES", "mylib", "1.0", short_commit_hash(spec)) + with open(os.path.join(source_dir, "hello.txt")) as fh: + return fh.read() + + def test_fresh_clone_is_patched(self): + self.assertEqual(self._checkout(), "hello patched\n") + + class SourceCheckoutValidityTest(unittest.TestCase): """_is_valid_source_checkout: a git source dir with no .git is not usable.""" From f0145c3d80f020b2cf660efa46da28b3a2175472 Mon Sep 17 00:00:00 2001 From: Maria Folina <139060140+m-fol@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:01:52 +0200 Subject: [PATCH 2/7] workarea: reapply patches on a reused git checkout --- bits_helpers/workarea.py | 3 +++ tests/test_workarea.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/bits_helpers/workarea.py b/bits_helpers/workarea.py index 9488d86b..c0e395b0 100644 --- a/bits_helpers/workarea.py +++ b/bits_helpers/workarea.py @@ -922,6 +922,9 @@ def _download_one(s): scm_exec(scm.fetchCmd(spec["source"], tag_ref), source_dir) scm_exec(scm.checkoutCmd(spec["tag"]), source_dir) _verify_commit_pin(scm, spec, source_dir, enforce_mode) + for sentinel in (".bits_patched", ".bits_applied_patches"): + if os.path.exists(os.path.join(source_dir, sentinel)): + os.remove(os.path.join(source_dir, sentinel)) _stage_patches() _apply_patches(spec, source_dir) else: diff --git a/tests/test_workarea.py b/tests/test_workarea.py index 9e8148e6..dee75669 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -409,6 +409,10 @@ def _checkout(self): def test_fresh_clone_is_patched(self): self.assertEqual(self._checkout(), "hello patched\n") + def test_existing_checkout_is_patched_again(self): + self._checkout() + self.assertEqual(self._checkout(), "hello patched\n") + class SourceCheckoutValidityTest(unittest.TestCase): """_is_valid_source_checkout: a git source dir with no .git is not usable.""" From f63c21149c9e1681833fe505445222121ddd3414 Mon Sep 17 00:00:00 2001 From: Maria Folina <139060140+m-fol@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:01:42 +0200 Subject: [PATCH 3/7] fix: clean checkout reuse before reapplying patches --- bits_helpers/git.py | 3 +++ bits_helpers/scm.py | 2 ++ bits_helpers/sl.py | 3 +++ bits_helpers/workarea.py | 4 +--- tests/test_workarea.py | 25 +++++++++++++++++++------ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/bits_helpers/git.py b/bits_helpers/git.py index 8bb0d2d8..324483d6 100644 --- a/bits_helpers/git.py +++ b/bits_helpers/git.py @@ -70,6 +70,9 @@ def cloneSourceCmd(self, source, destination, referenceRepo, usePartialClone): def checkoutCmd(self, tag): return ["checkout", "-f", tag] + def cleanCmd(self): + return ["clean", "-fdxq"] + def fetchCmd(self, remote, *refs): return ["fetch", "-f", "--prune"] + clone_speedup_options() + [remote, *refs] diff --git a/bits_helpers/scm.py b/bits_helpers/scm.py index bea737b8..83946692 100644 --- a/bits_helpers/scm.py +++ b/bits_helpers/scm.py @@ -18,6 +18,8 @@ def exec(self, *args, **kwargs): raise NotImplementedError def checkoutCmd(self, tag): raise NotImplementedError + def cleanCmd(self): + raise NotImplementedError def fetchCmd(self, remote, *refs): raise NotImplementedError def cloneReferenceCmd(self, spec, referenceRepo, usePartialClone): diff --git a/bits_helpers/sl.py b/bits_helpers/sl.py index 51adc43f..ef15d8de 100644 --- a/bits_helpers/sl.py +++ b/bits_helpers/sl.py @@ -52,6 +52,9 @@ def parseRefs(self, output): def listRefsCmd(self, repository): return ["bookmark", "--list", "--remote", "-R", repository] + def cleanCmd(self): + return ["clean", "--all"] + def diffCmd(self, directory): return "cd %s && sl diff && sl status" % directory diff --git a/bits_helpers/workarea.py b/bits_helpers/workarea.py index c0e395b0..d9cc17e9 100644 --- a/bits_helpers/workarea.py +++ b/bits_helpers/workarea.py @@ -922,9 +922,7 @@ def _download_one(s): scm_exec(scm.fetchCmd(spec["source"], tag_ref), source_dir) scm_exec(scm.checkoutCmd(spec["tag"]), source_dir) _verify_commit_pin(scm, spec, source_dir, enforce_mode) - for sentinel in (".bits_patched", ".bits_applied_patches"): - if os.path.exists(os.path.join(source_dir, sentinel)): - os.remove(os.path.join(source_dir, sentinel)) + scm_exec(scm.cleanCmd(), source_dir) _stage_patches() _apply_patches(spec, source_dir) else: diff --git a/tests/test_workarea.py b/tests/test_workarea.py index dee75669..07ea65d2 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -383,11 +383,13 @@ def setUp(self): subprocess.run(git + ["tag", "v1.0"], check=True) with open(os.path.join(self.pkgdir, "patches", "hello.patch"), "w") as fh: fh.write("--- a/hello.txt\n+++ b/hello.txt\n@@ -1 +1 @@\n-hello\n+hello patched\n") + with open(os.path.join(self.pkgdir, "patches", "new.patch"), "w") as fh: + fh.write("--- /dev/null\n+++ b/new.txt\n@@ -0,0 +1 @@\n+new\n") def tearDown(self): shutil.rmtree(self.tmp, ignore_errors=True) - def _checkout(self): + def _checkout(self, patches=("hello.patch",)): from bits_helpers.workarea import checkout_sources from bits_helpers.utilities import short_commit_hash spec = OrderedDict(( @@ -399,19 +401,30 @@ def _checkout(self): ("scm", Git()), ("is_devel_pkg", False), ("pkgdir", self.pkgdir), - ("patches", ["hello.patch"]), + ("patches", list(patches)), )) checkout_sources(spec, self.work_dir, None, False) - source_dir = os.path.join(self.work_dir, "SOURCES", "mylib", "1.0", short_commit_hash(spec)) - with open(os.path.join(source_dir, "hello.txt")) as fh: + return os.path.join(self.work_dir, "SOURCES", "mylib", "1.0", short_commit_hash(spec)) + + def _read(self, source_dir, name): + with open(os.path.join(source_dir, name)) as fh: return fh.read() def test_fresh_clone_is_patched(self): - self.assertEqual(self._checkout(), "hello patched\n") + self.assertEqual(self._read(self._checkout(), "hello.txt"), "hello patched\n") def test_existing_checkout_is_patched_again(self): self._checkout() - self.assertEqual(self._checkout(), "hello patched\n") + self.assertEqual(self._read(self._checkout(), "hello.txt"), "hello patched\n") + + def test_a_file_a_patch_creates_stays_on_every_build(self): + for _ in range(3): + self.assertEqual(self._read(self._checkout(["new.patch"]), "new.txt"), "new\n") + + def test_a_file_from_a_dropped_patch_is_removed(self): + self._checkout(["hello.patch", "new.patch"]) + source_dir = self._checkout(["hello.patch"]) + self.assertFalse(os.path.exists(os.path.join(source_dir, "new.txt"))) class SourceCheckoutValidityTest(unittest.TestCase): From 810997512eb8f1e9c676237ad90b41af5bf95a3d Mon Sep 17 00:00:00 2001 From: Maria Folina <139060140+m-fol@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:51:02 +0200 Subject: [PATCH 4/7] sl: clean ignored files with --ignored --- bits_helpers/sl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bits_helpers/sl.py b/bits_helpers/sl.py index ef15d8de..66af3611 100644 --- a/bits_helpers/sl.py +++ b/bits_helpers/sl.py @@ -53,7 +53,7 @@ def listRefsCmd(self, repository): return ["bookmark", "--list", "--remote", "-R", repository] def cleanCmd(self): - return ["clean", "--all"] + return ["clean", "--ignored"] def diffCmd(self, directory): return "cd %s && sl diff && sl status" % directory From 224d57b762aeb8042dbb85a28d4817f470ebed18 Mon Sep 17 00:00:00 2001 From: Maria Folina <139060140+m-fol@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:10:57 +0200 Subject: [PATCH 5/7] tests: what the clean removes and keeps on a reused checkout --- tests/test_workarea.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_workarea.py b/tests/test_workarea.py index 07ea65d2..c056f648 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -426,6 +426,32 @@ def test_a_file_from_a_dropped_patch_is_removed(self): source_dir = self._checkout(["hello.patch"]) self.assertFalse(os.path.exists(os.path.join(source_dir, "new.txt"))) + def test_a_package_without_patches_gets_a_clean_tree(self): + source_dir = self._checkout(()) + with open(os.path.join(source_dir, "leftover.txt"), "w") as fh: + fh.write("from the last build\n") + self._checkout(()) + self.assertFalse(os.path.exists(os.path.join(source_dir, "leftover.txt"))) + self.assertEqual(self._read(source_dir, "hello.txt"), "hello\n") + + def test_an_ignored_file_is_removed(self): + source_dir = self._checkout() + with open(os.path.join(source_dir, ".git", "info", "exclude"), "a") as fh: + fh.write("*.gen\n") + with open(os.path.join(source_dir, "configure.gen"), "w") as fh: + fh.write("generated\n") + self._checkout() + self.assertFalse(os.path.exists(os.path.join(source_dir, "configure.gen"))) + + def test_a_nested_git_repository_is_kept(self): + source_dir = self._checkout() + sub = os.path.join(source_dir, "sub") + subprocess.run(["git", "init", "-q", sub], check=True) + with open(os.path.join(sub, "file.txt"), "w") as fh: + fh.write("submodule\n") + self._checkout() + self.assertEqual(self._read(sub, "file.txt"), "submodule\n") + class SourceCheckoutValidityTest(unittest.TestCase): """_is_valid_source_checkout: a git source dir with no .git is not usable.""" From 7e3c15fe02dfd82be7365afffff26da5e8ec8684 Mon Sep 17 00:00:00 2001 From: Maria Folina <139060140+m-fol@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:26:58 +0200 Subject: [PATCH 6/7] workarea: note why the clean is unconditional --- bits_helpers/workarea.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bits_helpers/workarea.py b/bits_helpers/workarea.py index d9cc17e9..64ecdd70 100644 --- a/bits_helpers/workarea.py +++ b/bits_helpers/workarea.py @@ -922,6 +922,7 @@ def _download_one(s): scm_exec(scm.fetchCmd(spec["source"], tag_ref), source_dir) scm_exec(scm.checkoutCmd(spec["tag"]), source_dir) _verify_commit_pin(scm, spec, source_dir, enforce_mode) + # clean even without patches, otherwise files added by a dropped patch stay around scm_exec(scm.cleanCmd(), source_dir) _stage_patches() _apply_patches(spec, source_dir) From fa8ca1383668baaeb831172777ff301f5f84f121 Mon Sep 17 00:00:00 2001 From: Maria Folina <139060140+m-fol@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:39:12 +0200 Subject: [PATCH 7/7] tests: say what each git source patch test checks --- tests/test_workarea.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/test_workarea.py b/tests/test_workarea.py index c056f648..a4ee8f2a 100644 --- a/tests/test_workarea.py +++ b/tests/test_workarea.py @@ -411,45 +411,61 @@ def _read(self, source_dir, name): return fh.read() def test_fresh_clone_is_patched(self): - self.assertEqual(self._read(self._checkout(), "hello.txt"), "hello patched\n") + """First build: the patch is applied after the clone.""" + self.assertEqual(self._read(self._checkout(), "hello.txt"), "hello patched\n", + "patch was not applied after cloning") def test_existing_checkout_is_patched_again(self): + """Rebuild: the reused checkout is reset and patched again.""" self._checkout() - self.assertEqual(self._read(self._checkout(), "hello.txt"), "hello patched\n") + self.assertEqual(self._read(self._checkout(), "hello.txt"), "hello patched\n", + "reused checkout was built without its patch") def test_a_file_a_patch_creates_stays_on_every_build(self): - for _ in range(3): - self.assertEqual(self._read(self._checkout(["new.patch"]), "new.txt"), "new\n") + """A file added by a patch is there on every build, not only the first.""" + for build in range(1, 4): + self.assertEqual(self._read(self._checkout(["new.patch"]), "new.txt"), "new\n", + "new.txt missing on build %d" % build) def test_a_file_from_a_dropped_patch_is_removed(self): + """Dropping a patch from the recipe also removes the file it added.""" self._checkout(["hello.patch", "new.patch"]) source_dir = self._checkout(["hello.patch"]) - self.assertFalse(os.path.exists(os.path.join(source_dir, "new.txt"))) + self.assertFalse(os.path.exists(os.path.join(source_dir, "new.txt")), + "new.txt from the dropped patch is still there") def test_a_package_without_patches_gets_a_clean_tree(self): + """No patches: leftovers from the last build are still removed.""" source_dir = self._checkout(()) with open(os.path.join(source_dir, "leftover.txt"), "w") as fh: fh.write("from the last build\n") self._checkout(()) - self.assertFalse(os.path.exists(os.path.join(source_dir, "leftover.txt"))) - self.assertEqual(self._read(source_dir, "hello.txt"), "hello\n") + self.assertFalse(os.path.exists(os.path.join(source_dir, "leftover.txt")), + "leftover.txt survived the rebuild") + self.assertEqual(self._read(source_dir, "hello.txt"), "hello\n", + "the source itself was changed") def test_an_ignored_file_is_removed(self): + """Files matched by .gitignore (e.g. generated ones) are removed too.""" source_dir = self._checkout() with open(os.path.join(source_dir, ".git", "info", "exclude"), "a") as fh: fh.write("*.gen\n") with open(os.path.join(source_dir, "configure.gen"), "w") as fh: fh.write("generated\n") self._checkout() - self.assertFalse(os.path.exists(os.path.join(source_dir, "configure.gen"))) + self.assertFalse(os.path.exists(os.path.join(source_dir, "configure.gen")), + "ignored file survived the rebuild (clean without -x?)") def test_a_nested_git_repository_is_kept(self): + """A nested repository, like a submodule, is left alone.""" source_dir = self._checkout() sub = os.path.join(source_dir, "sub") subprocess.run(["git", "init", "-q", sub], check=True) with open(os.path.join(sub, "file.txt"), "w") as fh: fh.write("submodule\n") self._checkout() + self.assertTrue(os.path.exists(os.path.join(sub, "file.txt")), + "nested repository was deleted (clean with -ff?)") self.assertEqual(self._read(sub, "file.txt"), "submodule\n")