mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-94187 Switch git ssh to https in copybara (#27211) 

GitOrigin-RevId: 872162766296a1908bda09c3845615e3e5dde576
This commit is contained in:
Mikhail Shchatko 2024-11-20 22:30:38 +02:00 committed by MongoDB Bot
parent c30eb1a854
commit f8bb4ea493
6 changed files with 235 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
OWNERS.yml
View File

@ -53,7 +53,8 @@ filters:
- "copy.bara.sky":
approvers:
- IamXander
- "copybara.staging.sky":
- smcclure15
- "copy.bara.staging.sky":
approvers:
- devprod-correctness
- "jsconfig.json":

273
buildscripts/sync_repo_with_copybara.py
View File

@ -2,6 +2,7 @@
from __future__ import annotations
import argparse
import fileinput
import subprocess
import os
import sys
@ -16,41 +17,77 @@ from buildscripts.util.read_config import read_config_file
from evergreen.api import RetryingEvergreenApi
class CopybaraBranchNames(NamedTuple):
"""Copybara sync branch names."""
class CopybaraRepoConfig(NamedTuple):
"""Copybara source and destination repo sync configuration."""
source: Optional[str]
destination: Optional[str]
git_url: Optional[str] = None
repo_name: Optional[str] = None
branch: Optional[str] = None
class CopybaraConfig(NamedTuple):
"""Copybara sync configuration."""
source: Optional[CopybaraRepoConfig] = None
destination: Optional[CopybaraRepoConfig] = None
@classmethod
def empty(cls) -> CopybaraBranchNames:
def empty(cls) -> CopybaraConfig:
return cls(
source=None,
destination=None,
)
@classmethod
def from_copybara_sky_file(cls, file_path: str) -> CopybaraBranchNames:
source_branch_name_pattern = re.compile(r'ref = "(.+?)"')
destination_branch_name_pattern = re.compile(r'push = "(.+?)"')
with open(file_path, 'r') as file:
def from_copybara_sky_file(cls, file_path: str) -> CopybaraConfig:
with open(file_path, "r") as file:
content = file.read()
source_branch_name_match = source_branch_name_pattern.search(content)
destination_branch_name_match = destination_branch_name_pattern.search(content)
# Delete comments
content = re.sub(r"#.*", "", content)
# Check if both required patterns were found and return them
if source_branch_name_match and destination_branch_name_match:
return cls(
source=source_branch_name_match.group(1),
destination=destination_branch_name_match.group(1),
)
# Return empty if any pattern is not found
source_url_match = re.search(r'sourceUrl = "(.+?)"', content)
if source_url_match is None:
return cls.empty()
source_branch_name_match = re.search(r'ref = "(.+?)"', content)
if source_branch_name_match is None:
return cls.empty()
destination_url_match = re.search(r'destinationUrl = "(.+?)"', content)
if destination_url_match is None:
return cls.empty()
destination_branch_name_match = re.search(r'push = "(.+?)"', content)
if destination_branch_name_match is None:
return cls.empty()
repo_name_regex = re.compile(r"([^:/]+/[^:/]+)\.git")
source_git_url = source_url_match.group(1)
source_repo_name_match = repo_name_regex.search(source_git_url)
if source_repo_name_match is None:
return cls.empty()
destination_git_url = destination_url_match.group(1)
destination_repo_name_match = repo_name_regex.search(destination_git_url)
if destination_repo_name_match is None:
return cls.empty()
return cls(
source=CopybaraRepoConfig(
git_url=source_git_url,
repo_name=source_repo_name_match.group(1),
branch=source_branch_name_match.group(1),
),
destination=CopybaraRepoConfig(
git_url=destination_git_url,
repo_name=destination_repo_name_match.group(1),
branch=destination_branch_name_match.group(1),
),
)
def is_complete(self) -> bool:
return self.source and self.destination
return self.source is not None and self.destination is not None
def run_command(command):
@ -135,51 +172,51 @@ def send_failure_message_to_slack(expansions):
)
def check_destination_branch_exists(repo_url: str, branch_name: str) -> bool:
def check_destination_branch_exists(copybara_config: CopybaraConfig) -> bool:
"""
Check if a specific branch exists in a remote git repository.
Check if a specific branch exists in the destination git repository.
Args:
- repo_url (str): The URL of the remote git repository.
- branch_name (str): The name of the branch to check for existence.
- copybara_config (CopybaraConfig): Copybara configuration.
Returns
- bool: `True` if the branch exists in the remote repository, `False` otherwise.
- bool: `True` if the branch exists in the destination repository, `False` otherwise.
"""
command = f"git ls-remote {repo_url} {branch_name}"
command = (
f"git ls-remote {copybara_config.destination.git_url} {copybara_config.destination.branch}")
output = run_command(command)
return branch_name in output
return copybara_config.destination.branch in output
def find_matching_commit(dir_10gen_mongo: str, dir_mongodb_mongo: str) -> str:
def find_matching_commit(dir_source_repo: str, dir_destination_repo: str) -> Optional[str]:
"""
Finds a matching commit in the dir_mongodb_mongo repository based on the private hash from the dir_10gen_mongo repository.
Finds a matching commit in the destination repository based on the commit hash from the source repository.
Args:
- dir_10gen_mongo: The directory of the 10gen-mongo repository.
- dir_mongodb_mongo: The directory of the mongodb-mongo repository.
- dir_source_repo: The directory of the source repository.
- dir_destination_repo: The directory of the destination repository.
Returns
The hash of the matching commit if found; otherwise, prints a message and returns None.
"""
# Navigate to the 10gen-mongo repository
os.chdir(dir_10gen_mongo)
# Navigate to the source repository
os.chdir(dir_source_repo)
# Find the latest commit hash.
private_hash = run_command("git log --pretty=format:\"%H\" -1")
source_hash = run_command('git log --pretty=format:"%H" -1')
# Attempt to find a matching commit in the mongodb-mongo repository.
# Attempt to find a matching commit in the destination repository.
commit = run_command(
f"git --git-dir={dir_mongodb_mongo}/.git log -1 --pretty=format:\"%H\" --grep \"GitOrigin-RevId: {private_hash}\""
f'git --git-dir={dir_destination_repo}/.git log -1 --pretty=format:"%H" --grep "GitOrigin-RevId: {source_hash}"'
)
first_commit = run_command("git rev-list --max-parents=0 HEAD")
# Loop until a matching commit is found or the first commit is reached.
while len(commit.splitlines()) != 1:
current_commit = run_command("git log --pretty=format:\"%H\" -1")
current_commit = run_command('git log --pretty=format:"%H" -1')
if current_commit.strip() == first_commit.strip():
print(
@ -187,84 +224,82 @@ def find_matching_commit(dir_10gen_mongo: str, dir_mongodb_mongo: str) -> str:
)
return None
# Revert to the previous commit in the 10gen-mongo repository and try again.
# Revert to the previous commit in the source repository and try again.
run_command("git checkout HEAD~1")
private_hash = run_command("git log --pretty=format:\"%H\" -1")
source_hash = run_command('git log --pretty=format:"%H" -1')
# Attempt to find a matching commit again in the mongodb-mongo repository.
# Attempt to find a matching commit again in the destination repository.
commit = run_command(
f"git --git-dir={dir_mongodb_mongo}/.git log -1 --pretty=format:\"%H\" --grep \"GitOrigin-RevId: {private_hash}\""
f'git --git-dir={dir_destination_repo}/.git log -1 --pretty=format:"%H" --grep "GitOrigin-RevId: {source_hash}"'
)
return commit
def is_only_mongodb_mongo_repo():
def has_only_destination_repo_remote(repo_name: str):
"""
Check if the current directory's Git repository only contains the 'mongodb/mongo.git' remote URL.
Check if the current directory's Git repository only contains the destination repository remote URL.
Returns
bool: True if the repository only contains the 'mongodb/mongo.git' remote URL, False otherwise.
bool: True if the repository only contains the destination repository remote URL, False otherwise.
"""
git_config_path = os.path.join('.git', 'config')
with open(git_config_path, 'r') as f:
config_content = f.read()
# Define a regular expression pattern to match the mongodb/mongo.git
url_pattern = r'url\s*=\s*(.*?\.git\s*)'
# Define a regular expression pattern to match the '{owner}/{repo}.git'
url_pattern = r"url\s*=\s*(.*?\.git\s*)"
matches = re.findall(url_pattern, config_content)
if len(matches) == 1 and matches[0].strip().endswith('mongodb/mongo.git'):
if len(matches) == 1 and matches[0].strip().endswith(f"{repo_name}.git"):
return True
print(
"The current directory's Git repository does not only contain the 'mongodb/mongo.git' remote URL."
f"The current directory's Git repository contains not only the '{repo_name}.git' remote URL."
)
return False
def push_branch_to_public_repo(mongodb_mongo_dir: str, repo_url: str, destination_branch_name: str,
def push_branch_to_destination_repo(destination_repo_dir: str, copybara_config: CopybaraConfig,
branching_off_commit: str):
"""
Pushes a new branch to the remote repository after ensuring it branches off the public repository.
Args:
mongodb_mongo_dir (str): Path to the cloned 'mongodb-mongo' repository.
repo_url (str): The URL of the remote repository where the new branch will be pushed.
destination_branch_name (str): The name for the new branch to be pushed to the remote repository.
commit (str): The commit hash of the matching commit in the 'mongodb-mongo' repository.
destination_repo_dir (str): Path to the cloned destination repository.
copybara_config (CopybaraConfig): Copybara configuration.
branching_off_commit (str): The commit hash of the matching commit in the destination repository.
Raises
Exception: If the new branch is not branching off the public repository.
Exception: If the new branch is not branching off the destination repository.
"""
os.chdir(mongodb_mongo_dir)
os.chdir(destination_repo_dir)
# Check the current repo is only mongodb/mongo repo.
if not is_only_mongodb_mongo_repo():
raise Exception("Not only mongodb repo")
# Check the current repo has only destination repository remote.
if not has_only_destination_repo_remote(copybara_config.destination.repo_name):
raise Exception(f"{destination_repo_dir} git repo has not only the destination repo remote")
# Confirm the top commit is matching the found commit before pushing
new_branch_top_commit = run_command("git log --pretty=format:\"%H\" -1")
new_branch_top_commit = run_command('git log --pretty=format:"%H" -1')
if not new_branch_top_commit == branching_off_commit:
raise Exception(
"The new branch top commit does not match the branching_off_commit. Aborting push.")
# Confirming whether the commit exists in the remote public repository (mongodb/mongo/master) to ensure we are not pushing anything that isn't already in the public repository.
# run_command will raise an exception if the commit is not found in the remote branch.
# Confirming whether the commit exists in the destination repository to ensure
# we are not pushing anything that isn't already in the destination repository.
# run_command will raise an exception if the commit is not found in the destination branch.
run_command(f"git branch -r --contains {new_branch_top_commit}")
# Push the new branch to the remote repository
run_command(f"git push {repo_url} {destination_branch_name}")
# Push the new branch to the destination repository
run_command(
f"git push {copybara_config.destination.git_url} {copybara_config.destination.branch}")
def create_branch_from_matching_commit(repo_url: str, source_branch_name: str,
destination_branch_name: str) -> None:
def create_branch_from_matching_commit(copybara_config: CopybaraConfig) -> None:
"""
Creates a new branch in the mongodb/mongo repository based on a matching commit found in 10gen/mongo/branches and mongodb/mongo/master.
Create a new branch in the copybara destination repository based on a matching commit found in source repository and destination repository.
Args:
repo_url (str): The URL of the remote repository where the new branch will be pushed.
source_branch_name (str): The name of the branch in the '10gen-mongo' repository.
destination_branch_name (str): The name for the new branch to be created in the 'mongodb-mongo' repository.
copybara_config (CopybaraConfig): Copybara configuration.
"""
# Save original dirtory
@ -277,33 +312,33 @@ def create_branch_from_matching_commit(repo_url: str, source_branch_name: str,
os.makedirs(working_dir, exist_ok=True)
os.chdir(working_dir)
# Clone the specified branch of the 10gen/mongo and master of mongodb/mongo repo
cloned_10gen_mongo_dir = os.path.join(working_dir, "10gen-mongo")
cloned_mongodb_mongo_dir = os.path.join(working_dir, "mongodb-mongo")
# Clone the specified branch of the source repository and master of destination repository
cloned_source_repo_dir = os.path.join(working_dir, "source-repo")
cloned_destination_repo_dir = os.path.join(working_dir, "destination-repo")
run_command(f"git clone -b {copybara_config.source.branch}"
f" {copybara_config.source.git_url} {cloned_source_repo_dir}")
run_command(
f"git clone -b {source_branch_name} git@github.com:10gen/mongo.git {cloned_10gen_mongo_dir}"
)
run_command(f"git clone git@github.com:mongodb/mongo.git {cloned_mongodb_mongo_dir}")
f"git clone {copybara_config.destination.git_url} {cloned_destination_repo_dir}")
# Find matching commits to branching off
commit = find_matching_commit(cloned_10gen_mongo_dir, cloned_mongodb_mongo_dir)
commit = find_matching_commit(cloned_source_repo_dir, cloned_destination_repo_dir)
if commit is not None:
# Delete the cloned_10gen_mongo_dir folder
shutil.rmtree(cloned_10gen_mongo_dir)
if os.path.exists(cloned_10gen_mongo_dir):
raise Exception(cloned_10gen_mongo_dir + ": did not get removed")
# Delete the cloned_source_repo_dir folder
shutil.rmtree(cloned_source_repo_dir)
if os.path.exists(cloned_source_repo_dir):
raise Exception(cloned_source_repo_dir + ": did not get removed")
# Once a matching commit is found, create a new branch based on it.
os.chdir(cloned_mongodb_mongo_dir)
run_command(f"git checkout -b {destination_branch_name} {commit}")
os.chdir(cloned_destination_repo_dir)
run_command(f"git checkout -b {copybara_config.destination.branch} {commit}")
# Push the new branch to the remote repository
push_branch_to_public_repo(cloned_mongodb_mongo_dir, repo_url, destination_branch_name,
commit)
push_branch_to_destination_repo(cloned_destination_repo_dir, copybara_config, commit)
else:
print(
f"Could not find matching commits between mongodb/mongo/master and 10gen/mongo/{destination_branch_name} to branching off"
f"Could not find matching commits between {copybara_config.destination.repo_name}/master"
f" and {copybara_config.source.repo_name}/{copybara_config.source.branch} to branching off"
)
sys.exit(1)
except Exception as err:
@ -335,39 +370,65 @@ def main():
# Read configurations
expansions = read_config_file(args.expansions_file)
access_token_copybara_syncer = get_installation_access_token(
expansions["app_id_copybara_syncer"], expansions["private_key_copybara_syncer"],
expansions["installation_id_copybara_syncer"])
token_mongodb_mongo = get_installation_access_token(
expansions["app_id_copybara_syncer"],
expansions["private_key_copybara_syncer"],
expansions["installation_id_copybara_syncer"],
)
token_10gen_mongo = get_installation_access_token(
expansions["app_id_copybara_syncer_10gen"],
expansions["private_key_copybara_syncer_10gen"],
expansions["installation_id_copybara_syncer_10gen"],
)
tokens_map = {
"https://github.com/mongodb/mongo.git": token_mongodb_mongo,
"https://github.com/10gen/mongo.git": token_10gen_mongo,
}
# Create the mongodb-bot.gitconfig file as necessary.
create_mongodb_bot_gitconfig()
current_dir = os.getcwd()
config_file = f"{current_dir}/copy.bara.sky"
git_destination_url_with_token = f"https://x-access-token:{access_token_copybara_syncer}@github.com/mongodb/mongo.git"
# create destination branch if not exists
copybara_branch_names = CopybaraBranchNames.from_copybara_sky_file(config_file)
if not copybara_branch_names.is_complete():
# Overwrite repo urls in copybara config in-place
with fileinput.FileInput(config_file, inplace=True) as file:
for line in file:
token = None
for repo, value in tokens_map.items():
if repo in line:
token = value
if token:
print(
line.replace(
"https://github.com",
f"https://x-access-token:{token}@github.com",
),
end="",
)
else:
print(line, end="")
copybara_config = CopybaraConfig.from_copybara_sky_file(config_file)
# Create destination branch if it does not exist
if not copybara_config.is_complete():
print("ERROR!!!")
print(
f"ERROR!!! Source or destination branch name could not be parsed from the {config_file}."
f"ERROR!!! Source or destination configuration could not be parsed from the {config_file}."
)
print("ERROR!!!")
sys.exit(1)
else:
if not check_destination_branch_exists(git_destination_url_with_token,
copybara_branch_names.destination):
create_branch_from_matching_commit(git_destination_url_with_token,
copybara_branch_names.source,
copybara_branch_names.destination)
print(
f"New branch named {copybara_branch_names.destination} has been created for the mongodb/mongo repo"
)
if not check_destination_branch_exists(copybara_config):
create_branch_from_matching_commit(copybara_config)
print(f"New branch named '{copybara_config.destination.branch}' has been created"
f" for the '{copybara_config.destination.repo_name}' repo")
else:
print(
f"The branch named {copybara_branch_names.destination} already exists in the mongodb/mongo repo."
)
print(f"The branch named '{copybara_config.destination.branch}' already exists"
f" in the '{copybara_config.destination.repo_name}' repo.")
# Set up the Docker command and execute it
docker_cmd = [
@ -377,7 +438,7 @@ def main():
f'-v "{config_file}":/usr/src/app/copy.bara.sky',
"-e COPYBARA_CONFIG='copy.bara.sky'",
"-e COPYBARA_SUBCOMMAND='migrate'",
f"-e COPYBARA_OPTIONS='-v --git-destination-url={git_destination_url_with_token}'",
"-e COPYBARA_OPTIONS='-v'",
"copybara copybara",
]

67
buildscripts/tests/test_sync_repo_with_copybara.py
View File

@ -3,7 +3,6 @@ import os
import sys
import tempfile
import traceback
from unittest.mock import patch
from buildscripts import sync_repo_with_copybara
@ -126,17 +125,27 @@ class TestBranchFunctions(unittest.TestCase):
def test_branch_exists(self):
"""Perform a test to check that the branch exists in a repository."""
test_name = "branch_exists_test"
repo_url = "git@github.com:mongodb/mongo.git"
branch_name = "v7.3"
result = sync_repo_with_copybara.check_destination_branch_exists(repo_url, branch_name)
copybara_config = sync_repo_with_copybara.CopybaraConfig(
source=None,
destination=sync_repo_with_copybara.CopybaraRepoConfig(
git_url="https://github.com/mongodb/mongo.git",
branch="v7.3",
),
)
result = sync_repo_with_copybara.check_destination_branch_exists(copybara_config)
self.assertTrue(result, f"{test_name}: SUCCESS!")
def test_branch_not_exists(self):
"""Perform a test to check that the branch does not exists in a repository."""
"""Perform a test to check that the branch does not exist in a repository."""
test_name = "branch_not_exists_test"
repo_url = "git@github.com:mongodb/mongo.git"
branch_name = "v7.3test"
result = sync_repo_with_copybara.check_destination_branch_exists(repo_url, branch_name)
copybara_config = sync_repo_with_copybara.CopybaraConfig(
source=None,
destination=sync_repo_with_copybara.CopybaraRepoConfig(
git_url="https://github.com/mongodb/mongo.git",
branch="..invalid-therefore-impossible-to-create-branch-name",
),
)
result = sync_repo_with_copybara.check_destination_branch_exists(copybara_config)
self.assertFalse(result, f"{test_name}: SUCCESS!")
def test_only_mongodb_mongo_repo(self):
@ -155,7 +164,7 @@ class TestBranchFunctions(unittest.TestCase):
try:
# Check if the repository is only the MongoDB official repository
result = sync_repo_with_copybara.is_only_mongodb_mongo_repo()
result = sync_repo_with_copybara.has_only_destination_repo_remote("mongodb/mongo")
except Exception as err:
print(f"{test_name}: FAIL!\n Exception occurred: {err}\n {traceback.format_exc()}")
self.fail(f"{test_name}: FAIL!")
@ -180,11 +189,25 @@ class TestBranchFunctions(unittest.TestCase):
try:
# Call function to push branch to public repository, expecting an exception
sync_repo_with_copybara.push_branch_to_public_repo(mongodb_mongo_dir, repo_url='',
destination_branch_name='',
branching_off_commit='')
sync_repo_with_copybara.push_branch_to_destination_repo(
mongodb_mongo_dir,
copybara_config=sync_repo_with_copybara.CopybaraConfig(
source=sync_repo_with_copybara.CopybaraRepoConfig(
git_url="",
repo_name="",
branch="",
),
destination=sync_repo_with_copybara.CopybaraRepoConfig(
git_url="",
repo_name="",
branch="",
),
),
branching_off_commit="",
)
except Exception as err:
if str(err) == "Not only mongodb repo":
if (str(err) ==
f"{mongodb_mongo_dir} git repo has not only the destination repo remote"):
return
self.fail(f"{test_name}: FAIL!")
@ -211,8 +234,22 @@ class TestBranchFunctions(unittest.TestCase):
self.create_mock_repo_commits(mongodb_mongo_dir, 2)
try:
# Call function to push branch to public repository, expecting an exception
sync_repo_with_copybara.push_branch_to_public_repo(mongodb_mongo_dir, '', '',
invalid_branching_off_commit)
sync_repo_with_copybara.push_branch_to_destination_repo(
mongodb_mongo_dir,
sync_repo_with_copybara.CopybaraConfig(
source=sync_repo_with_copybara.CopybaraRepoConfig(
git_url="",
repo_name="",
branch="",
),
destination=sync_repo_with_copybara.CopybaraRepoConfig(
git_url="",
repo_name="",
branch="",
),
),
invalid_branching_off_commit,
)
except Exception as err:
if str(
err

6
copy.bara.sky
View File

@ -5,8 +5,8 @@
# sourceUrl = "/path/to/source"
# destinationUrl = "/path/to/dest"
sourceUrl = "git@github.com:10gen/mongo.git"
destinationUrl = "git@github.com:mongodb/mongo.git"
sourceUrl = "https://github.com/10gen/mongo.git"
destinationUrl = "https://github.com/mongodb/mongo.git"
core.workflow(
name = "default",
@ -26,7 +26,7 @@ core.workflow(
# Change the path here to the folder you want to publish publicly
transformations = [
# (^.*?) - matches the first line (without the newline char)
# \n - matches the first newline (or nothing at all if there is no newline). If there is no match then nopthing happens
# \n - matches the first newline (or nothing at all if there is no newline). If there is no match then nothing happens
# ((\n|.)*) - matches everything after
# Overall, this copies only the first line of the commit rather than the body
metadata.scrubber("(^.*?)\n((\n|.)*)", replacement = "$1"),

15
copybara.staging.sky → copy.bara.staging.sky
View File

@ -1,14 +1,13 @@
# This configuration is for migrating code from one Git repository to another using Copybara.
# It selectively copies content, excluding specific paths and preserving authorship.
sourceUrl = "git@github.com:10gen/mongo.git"
destinationUrl = "git@github.com:10gen/mongo-copybara.git"
sourceUrl = "https://github.com/10gen/mongo.git"
destinationUrl = "https://github.com/10gen/mongo-copybara.git"
core.workflow(
name = "default",
origin = git.origin(
url = sourceUrl,
ref = "v8.0",
# VersionSelector
),
destination = git.destination(
url = destinationUrl,
@ -20,7 +19,11 @@ core.workflow(
authoring = authoring.pass_thru("MongoDB <mongodb@mongodb.com>"),
mode = "ITERATIVE",
# Change the path here to the folder you want to publish publicly
# transformations = [
# core.move("path/to/folder/you/want/exported", ""),
# ],
transformations = [
# (^.*?) - matches the first line (without the newline char)
# \n - matches the first newline (or nothing at all if there is no newline). If there is no match then nothing happens
# ((\n|.)*) - matches everything after
# Overall, this copies only the first line of the commit rather than the body
metadata.scrubber("(^.*?)\n((\n|.)*)", replacement = "$1"),
],
)

1
etc/evergreen_yml_components/tasks/misc_tasks.yml
View File

@ -1214,6 +1214,7 @@ tasks:
- name: sync_repo_with_copybara
tags: ["assigned_to_jira_team_devprod_correctness", "experimental"]
patchable: false
commands:
- command: manifest.load
- func: "git get project and add git tag"