Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions .github/scripts/gitlab-mirror.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ export function mirrorTarget(repo, rootGroup, sourceOrganization) {
targetPath: transformed ? `${path}-${repo.id}` : repo.name,
targetGroup, targetVisibility: repo.visibility === 'public' ? 'public' : 'private',
description: `Mirror of ${repo.clone_url}\nGitHub repository ID: ${repo.id}`,
ciSettings: repo.owner.login.toLowerCase() === 'lizardbyte-infrastructure' ? {
builds_access_level: 'disabled', shared_runners_enabled: false,
group_runners_enabled: false, auto_devops_enabled: false,
} : {},
};
}

Expand Down Expand Up @@ -118,13 +122,21 @@ export class GitLabMirrors {
throw new Error('Existing GitLab mirror identity does not match the source repository');
}

/** Secure non-public sources before transferring or allowing the Git push stage to proceed. */
/** Verify private visibility and infrastructure CI settings before transferring or pushing Git data. */
async secure(project, repo) {
if (repo.targetVisibility !== 'private') return;
const result = await this.request('PUT', `/projects/${project.id}`, {visibility: 'private'});
if (result.status !== 200 || result.data.id !== project.id || result.data.visibility !== 'private') {
const settings = {...repo.ciSettings, ...(repo.targetVisibility === 'private' ? {visibility: 'private'} : {})};
if (!Object.keys(settings).length) return;
if (Object.entries(settings).every(([key, value]) => project[key] === value)) return;
const result = await this.request('PUT', `/projects/${project.id}`, settings);
if (result.status !== 200 || result.data.id !== project.id) {
throw new Error('GitLab project safety settings verification failed');
}
if (repo.targetVisibility === 'private' && result.data.visibility !== 'private') {
throw new Error('GitLab privacy verification failed');
}
if (Object.entries(repo.ciSettings).some(([key, value]) => result.data[key] !== value)) {
throw new Error('GitLab CI/CD disable verification failed');
}
}

/** Resolve an existing mirror or a new-project namespace, preserving legacy GitLab project IDs. */
Expand Down
13 changes: 11 additions & 2 deletions .github/scripts/mirror-gitlab.sh
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ for ((index = 0; index < repository_count; index++)); do
target_path="$(jq -er '.targetPath' <<< "${repository}")"
target_visibility="$(jq -er '.targetVisibility' <<< "${repository}")"
description="$(jq -er '.description' <<< "${repository}")"
ci_settings="$(jq -ec '.ciSettings' <<< "${repository}")"
group_id="$(jq -r '.namespaceId // empty' <<< "${repository}")"
target_group="$(jq -er '.targetGroup' <<< "${repository}")"
preparation_error="$(jq -r '.preparationError // empty' <<< "${repository}")"
Expand Down Expand Up @@ -196,6 +197,7 @@ for ((index = 0; index < repository_count; index++)); do
payload="$(
jq -nc \
--arg description "${description}" \
--argjson ci_settings "${ci_settings}" \
--arg import_url "${create_import_url}" \
--arg name "${target_name}" \
--arg path "${target_path}" \
Expand All @@ -208,7 +210,7 @@ for ((index = 0; index < repository_count; index++)); do
namespace_id: $namespace_id,
visibility: $visibility,
initialize_with_readme: false
} + if $import_url == "" then {} else {import_url: $import_url} end'
} + $ci_settings + if $import_url == "" then {} else {import_url: $import_url} end'
)"
if ! status="$(gitlab_request POST "${GITLAB_API_URL}/projects" "${payload}")"; then
echo "::error title=Mirror failed: ${source_name}::Unable to create the GitLab project." >&2
Expand Down Expand Up @@ -269,7 +271,8 @@ for ((index = 0; index < repository_count; index++)); do
jq -nc \
--arg description "${description}" \
--arg visibility "${target_visibility}" \
'{description: $description, visibility: $visibility}'
--argjson ci_settings "${ci_settings}" \
'{description: $description, visibility: $visibility} + $ci_settings'
)"
if ! status="$(
gitlab_request PUT "${GITLAB_API_URL}/projects/${project_id}" "${payload}"
Expand All @@ -293,6 +296,12 @@ for ((index = 0; index < repository_count; index++)); do
echo "::error title=Mirror failed: ${source_name}::GitLab visibility verification failed." >&2
exit 1
fi
if ! jq -e --argjson ci_settings "${ci_settings}" \
'. as $project | $ci_settings | to_entries | all(.[]; $project[.key] == .value)' \
"${response_file}" > /dev/null; then
echo "::error title=Mirror failed: ${source_name}::GitLab CI/CD disable verification failed." >&2
exit 1
fi

target_clone_url="$(jq -er '.ssh_url_to_repo' "${response_file}")"

Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@ and incomplete transfers stop that repository's delivery while other repositorie
verified as private before any transfer or Git push. Existing archived GitLab mirrors require manual handling;
the workflow does not unarchive them. Large-project imports and the existing GitLab SSH transport remain in use.

GitLab copies of `LizardByte-infrastructure` repositories have CI/CD, instance runners, group runners, and
Auto DevOps disabled. New projects receive these settings during creation, before any server-side import;
existing projects are updated during preparation and the settings are verified again before pushing. This
prevents mirrored upstream `.gitlab-ci.yml` files from starting GitLab jobs. Other source organizations retain
their existing CI settings. Disabling CI/CD does not delete pipeline history or cancel jobs that are already
running; cancel those jobs separately if needed.

Namespace preparation and mirror delivery share the same GitLab request helper. HTTP 429 responses are retried
up to five attempts, honoring `Retry-After` with exponential backoff when the header is absent. A requested wait
longer than five minutes is reported instead of retried early. Other HTTP errors, including 404s after project
Expand Down
73 changes: 73 additions & 0 deletions tests/gitlab-mirror.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,76 @@ test('target paths retain legacy spelling and stable normalization, with separat
assert.throws(() => mirrorTarget({clone_url: 'bad'}, 'lizardbyte', 'LizardByte'), /Invalid mirror source/);
});

const infrastructureGroup = {...group, full_path: 'lizardbyte/lizardbyte-infrastructure'};
const infrastructureProject = {id: 42, path_with_namespace: `${infrastructureGroup.full_path}/mirror`,
namespace: {id: 2}, description: 'GitHub repository ID: 99'};
const infrastructureRoutes = {
'GET /groups/lizardbyte': [200, root],
'GET /groups/lizardbyte%2Flizardbyte-infrastructure': [200, infrastructureGroup],
'GET /projects/lizardbyte%2Flizardbyte-infrastructure%2Fmirror': [200, infrastructureProject],
};
const infrastructureSettings = {
builds_access_level: 'disabled', shared_runners_enabled: false, group_runners_enabled: false, auto_devops_enabled: false,
};

test('only infrastructure mirror targets disable CI/CD, instance/group runners, and Auto DevOps', () => {
for (const owner of ['LizardByte-infrastructure', 'lizardbyte-INFRASTRUCTURE']) {
assert.deepEqual(target(owner).ciSettings, infrastructureSettings);
}
for (const owner of ['LizardByte', 'SunshineStream', 'Themerr', 'KokoStreamTV', 'Other']) {
assert.deepEqual(target(owner).ciSettings, {});
}
});

test('existing infrastructure mirrors have CI disabled and verified before delivery is allowed', async () => {
const {client, calls} = harness({...infrastructureRoutes,
'PUT /projects/42': [200, {...infrastructureProject, ...infrastructureSettings}]});
const result = await client.prepare(target('LizardByte-infrastructure', 'mirror'));
assert.equal(result.projectId, 42);
assert.deepEqual(calls.find(call => call.key === 'PUT /projects/42').body, infrastructureSettings);
});

test('infrastructure CI settings and private visibility are secured in the same verified update', async () => {
const {client, calls} = harness({...infrastructureRoutes,
'PUT /projects/42': [200, {...infrastructureProject, ...infrastructureSettings, visibility: 'private'}]});
await client.prepare(target('LizardByte-infrastructure', 'mirror', 99, 'private'));
assert.deepEqual(calls.find(call => call.key === 'PUT /projects/42').body, {...infrastructureSettings, visibility: 'private'});
});

test('denied or incompletely applied infrastructure CI settings stop delivery', async () => {
const responses = [[403, {}], [200, {...infrastructureProject, ...infrastructureSettings, id: 123}],
[200, infrastructureProject], ...Object.keys(infrastructureSettings).map(key =>
[200, {...infrastructureProject, ...infrastructureSettings, [key]: key === 'builds_access_level' ? 'enabled' : true}])];
for (const response of responses) {
const {client} = harness({...infrastructureRoutes, 'PUT /projects/42': response});
await assert.rejects(client.prepare(target('LizardByte-infrastructure', 'mirror')), /settings verification|CI\/CD disable/);
}
});

test('already disabled infrastructure mirrors avoid redundant preparation updates', async () => {
const {client, calls} = harness({...infrastructureRoutes,
'GET /projects/lizardbyte%2Flizardbyte-infrastructure%2Fmirror': [200, {...infrastructureProject, ...infrastructureSettings}]});
assert.equal((await client.prepare(target('LizardByte-infrastructure', 'mirror'))).projectId, 42);
assert.ok(calls.every(call => call.key.startsWith('GET')));
});

test('mirrors from other organizations preserve manually configured GitLab CI settings', async () => {
const project = {id: 42, path_with_namespace: 'lizardbyte/sunshinestream/Sunshine', namespace: {id: 2},
description: 'GitHub repository ID: 99', builds_access_level: 'private', shared_runners_enabled: true};
const {client, calls} = harness({...base, 'GET /projects/lizardbyte%2Fsunshinestream%2FSunshine': [200, project]});
assert.equal((await client.prepare(target())).projectId, 42);
assert.ok(calls.every(call => call.key.startsWith('GET')));
});

test('infrastructure mirror transfers cannot proceed until CI suppression is verified', async () => {
const legacy = {...infrastructureProject, path_with_namespace: 'lizardbyte/mirror', namespace: {id: 1}};
const {client, calls} = harness({...infrastructureRoutes,
'GET /projects/lizardbyte%2Flizardbyte-infrastructure%2Fmirror': [404, {}],
'GET /projects/lizardbyte%2Fmirror': [200, legacy], 'PUT /projects/42': [200, legacy]});
await assert.rejects(client.prepare(target('LizardByte-infrastructure', 'mirror')), /CI\/CD disable/);
assert.ok(!calls.some(call => call.key.endsWith('/transfer')));
});

test('mirror preparation rejects CLI paths before reading inventory or making requests', () => {
const result = spawnSync(process.execPath, ['.github/scripts/gitlab-mirror.mjs', '../unexpected.json'], {encoding: 'utf8'});
assert.equal(result.status, 1);
Expand Down Expand Up @@ -184,6 +254,9 @@ test('Git push remains behind preparation and private visibility verification, w
const script = readFileSync('.github/scripts/mirror-gitlab.sh', 'utf8');
assert.ok(script.indexOf('preparation_error}') < script.indexOf('Checking for the GitLab project'));
assert.ok(script.indexOf('GitLab privacy verification failed') < script.indexOf('clone --mirror'));
assert.ok(script.indexOf('GitLab CI/CD disable verification failed') < script.indexOf('clone --mirror'));
assert.match(script, /\} \+ \$ci_settings \+ if \$import_url/);
assert.match(script, /\{description: \$description, visibility: \$visibility\} \+ \$ci_settings/);
assert.match(script, /namespace_id/);
assert.match(script, /push --mirror/);
assert.match(script, /import\/git/);
Expand Down
Loading