Skip to content

build: update migration script with latest monorepo migration tasks #11977

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chingor13 wants to merge 24 commits into
base: main
Choose a base branch
from
Open

build: update migration script with latest monorepo migration tasks #11977

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
37cb7cb
build: create split integration Kokoro test config
chingor13 Feb 11, 2026
595bc9b
add module to excluded_modules for partial tests
chingor13 Feb 11, 2026
3ef8419
conditionally skip tests
chingor13 Feb 11, 2026
9caadae
prune managed dependencies
chingor13 Feb 11, 2026
4bf8432
fix dependencies in other submodules
chingor13 Feb 12, 2026
34912be
skip managed dependencies versions check for new split repos
chingor13 Feb 12, 2026
f2114b0
keep integration configs
chingor13 Feb 12, 2026
d2f742e
clean dependencies
chingor13 Feb 17, 2026
24639f4
fix build subdir spacing
chingor13 Feb 17, 2026
377a9c0
fix commit message when squashing commits
chingor13 Feb 19, 2026
8844b8f
rename customNative profile
chingor13 Feb 19, 2026
6052f5b
create split graalvm config
chingor13 Feb 19, 2026
7e54aff
conditionally add integration test args
chingor13 Feb 19, 2026
525d84c
accept EXTRA_KOKORO_ENVS
chingor13 Feb 19, 2026
cd30111
skip gapic-libraries-bom dependency management, manage all intra mono…
chingor13 Feb 19, 2026
d9bdaaa
skip managing boms for artifacts managed in monorepo
chingor13 Feb 20, 2026
7bbd426
update .repo-metadata repo fields
chingor13 Feb 23, 2026
8217b8e
move google-api-services dependency management
chingor13 Feb 24, 2026
a46f85a
ensure annotation-api in maven-dependency-plugin configuration
chingor13 Feb 24, 2026
4ff3cf7
add TEST_WITH_EXISTING env which allows easy testing with existing ko…
chingor13 Feb 24, 2026
7e00fb1
preserve appengine version management
chingor13 Feb 24, 2026
5621824
consolidate native test profiles
chingor13 Feb 25, 2026
48e464e
preserve kokoro custom scripts
chingor13 Mar 3, 2026
47e061d
helper to apply manual changes
chingor13 Mar 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 214 additions & 6 deletions monorepo-migration/migrate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ check_command() {
check_command git
check_command python3
check_command mvn
check_command jq

# Configuration
MONOREPO_URL="https://github.com/googleapis/google-cloud-java"
Expand Down Expand Up @@ -56,6 +57,7 @@ FIX_COPYRIGHT_SCRIPT="$TRANSFORM_SCRIPT_DIR/fix_copyright_headers.py"
UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config.py"
UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py"
TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py"
UPDATE_DEP_MGMT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_dependency_management.py"

# Track number of commits made by this script
COMMIT_COUNT=0
Expand Down Expand Up @@ -147,7 +149,8 @@ fi


# 2.5 Create a new feature branch for the migration
BRANCH_NAME="migrate-$SOURCE_REPO_NAME"
BRANCH_NAME_SUFFIX="${BRANCH_NAME_SUFFIX:-}"
BRANCH_NAME="migrate-${SOURCE_REPO_NAME}${BRANCH_NAME_SUFFIX}"
echo "Creating feature branch: $BRANCH_NAME"
if git rev-parse --verify "$BRANCH_NAME" >/dev/null 2>&1; then
git branch -D "$BRANCH_NAME"
Expand Down Expand Up @@ -180,7 +183,154 @@ rm -f "$SOURCE_REPO_NAME/renovate.json"
rm -f "$SOURCE_REPO_NAME/LICENSE"
rm -f "$SOURCE_REPO_NAME/java.header"
rm -rf "$SOURCE_REPO_NAME/.kokoro"
git checkout -- "$SOURCE_REPO_NAME/.kokoro/presubmit/*.sh" || true
# rm -rf "$SOURCE_REPO_NAME/.kokoro/continuous" "$SOURCE_REPO_NAME/.kokoro/nightly" "$SOURCE_REPO_NAME/.kokoro/presubmit"

# 6.6 Create split integration config if needed
SOURCE_INTEGRATION_CFG="$SOURCE_DIR/.kokoro/presubmit/integration.cfg"
if [ -f "$SOURCE_INTEGRATION_CFG" ]; then
echo "Creating split integration config for $SOURCE_REPO_NAME..."
SHORT_NAME="${SOURCE_REPO_NAME#java-}"
TARGET_INTEGRATION_CFG=".kokoro/presubmit/${SHORT_NAME}-integration.cfg"

cp "$SOURCE_INTEGRATION_CFG" "$TARGET_INTEGRATION_CFG"

# Replace JOB_TYPE with integration-single. Robustly handle multi-line or single-line.
perl -0777 -i -pe 's/(key:\s*"JOB_TYPE"\s*value:\s*")[^"]*(")/$1integration-single$2/g' "$TARGET_INTEGRATION_CFG"

# Append BUILD_SUBDIR
cat <<EOF >> "$TARGET_INTEGRATION_CFG"

env_vars: {
key: "BUILD_SUBDIR"
value: "$SOURCE_REPO_NAME"
}
EOF

if [ -n "$INTEGRATION_TEST_ARGS" ]; then
cat <<EOF >> "$TARGET_INTEGRATION_CFG"

env_vars: {
key: "INTEGRATION_TEST_ARGS"
value: "$INTEGRATION_TEST_ARGS"
}
EOF
fi

if [ -n "$EXTRA_KOKORO_ENVS" ]; then
OLD_IFS="$IFS"
IFS=','
for env in $EXTRA_KOKORO_ENVS; do
key="${env%%=*}"
value="${env#*=}"
cat <<EOF >> "$TARGET_INTEGRATION_CFG"

env_vars: {
key: "$key"
value: "$value"
}
EOF
done
IFS="$OLD_IFS"
fi

if [ "${TEST_WITH_EXISTING:-false}" = "true" ]; then
echo "Copying to logging-integration.cfg for testing..."
cp "$TARGET_INTEGRATION_CFG" ".kokoro/presubmit/logging-integration.cfg"
git add ".kokoro/presubmit/logging-integration.cfg"
fi

echo "Committing split integration config..."
git add "$TARGET_INTEGRATION_CFG"
git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split integration config"
COMMIT_COUNT=$((COMMIT_COUNT + 1))
fi

# 6.6b Create split GraalVM config if needed
SOURCE_GRAALVM_CFG="$SOURCE_DIR/.kokoro/presubmit/graalvm-native-a.cfg"
if [ -f "$SOURCE_GRAALVM_CFG" ]; then
echo "Creating split GraalVM config for $SOURCE_REPO_NAME..."
SHORT_NAME="${SOURCE_REPO_NAME#java-}"
TARGET_GRAALVM_CFG=".kokoro/presubmit/${SHORT_NAME}-graalvm-native-presubmit.cfg"

cp "$SOURCE_GRAALVM_CFG" "$TARGET_GRAALVM_CFG"

# Replace JOB_TYPE with graalvm-single. Robustly handle multi-line or single-line.
perl -0777 -i -pe 's/(key:\s*"JOB_TYPE"\s*value:\s*")[^"]*(")/$1graalvm-single$2/g' "$TARGET_GRAALVM_CFG"

# Append BUILD_SUBDIR
cat <<EOF >> "$TARGET_GRAALVM_CFG"

env_vars: {
key: "BUILD_SUBDIR"
value: "$SOURCE_REPO_NAME"
}
EOF

if [ -n "$INTEGRATION_TEST_ARGS" ]; then
cat <<EOF >> "$TARGET_GRAALVM_CFG"

env_vars: {
key: "INTEGRATION_TEST_ARGS"
value: "$INTEGRATION_TEST_ARGS"
}
EOF
fi

if [ -n "$EXTRA_KOKORO_ENVS" ]; then
OLD_IFS="$IFS"
IFS=','
for env in $EXTRA_KOKORO_ENVS; do
key="${env%%=*}"
value="${env#*=}"
cat <<EOF >> "$TARGET_GRAALVM_CFG"

env_vars: {
key: "$key"
value: "$value"
}
EOF
done
IFS="$OLD_IFS"
fi

if [ "${TEST_WITH_EXISTING:-false}" = "true" ]; then
echo "Copying to logging-graalvm-native-presubmit.cfg for testing..."
cp "$TARGET_GRAALVM_CFG" ".kokoro/presubmit/logging-graalvm-native-presubmit.cfg"
git add ".kokoro/presubmit/logging-graalvm-native-presubmit.cfg"
fi

echo "Committing split GraalVM config..."
git add "$TARGET_GRAALVM_CFG"
git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split GraalVM config"
COMMIT_COUNT=$((COMMIT_COUNT + 1))
fi

# 6.7 Update excluded_modules in .kokoro/common.sh
COMMON_SH=".kokoro/common.sh"
if [ -f "$COMMON_SH" ]; then
echo "Updating excluded_modules in $COMMON_SH..."
# Insert the new module name before the closing parenthesis of the excluded_modules array
sed -i "/^excluded_modules=(/,/^)/ s/^)/ '$SOURCE_REPO_NAME'\n)/" "$COMMON_SH"

echo "Committing excluded_modules update..."
git add "$COMMON_SH"
git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): add to excluded_modules in .kokoro/common.sh"
COMMIT_COUNT=$((COMMIT_COUNT + 1))
fi

# 6.8 Update .repo-metadata.json if it exists
REPO_METADATA="$SOURCE_REPO_NAME/.repo-metadata.json"
if [ -f "$REPO_METADATA" ]; then
echo "Updating $REPO_METADATA..."
# Update "repo" to googleapis/google-cloud-java and "repo_short" to google-cloud-java
jq '.repo = "googleapis/google-cloud-java" | .repo_short = "google-cloud-java"' "$REPO_METADATA" > "${REPO_METADATA}.tmp" && mv "${REPO_METADATA}.tmp" "$REPO_METADATA"

echo "Committing $REPO_METADATA update..."
git add "$REPO_METADATA"
git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): update .repo-metadata.json"
COMMIT_COUNT=$((COMMIT_COUNT + 1))
fi
rm -f "$SOURCE_REPO_NAME/codecov.yaml"
rm -f "$SOURCE_REPO_NAME/synth.metadata"
rm -f "$SOURCE_REPO_NAME/license-checks.xml"
Expand Down Expand Up @@ -243,6 +393,7 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then

# Cleanup empty .github directory if it exists
rm -rf "$SOURCE_REPO_NAME/.github"
git checkout -- "$SOURCE_REPO_NAME/.github/scripts" || true
git add -- "$SOURCE_REPO_NAME/.github"

echo "Committing workflow migration..."
Expand All @@ -266,6 +417,19 @@ if [ -f "$SOURCE_CONFIG" ]; then
COMMIT_COUNT=$((COMMIT_COUNT + 1))
fi

# 7.6b Update generation/check_non_release_please_versions.sh
CHECK_VERSIONS_SCRIPT="generation/check_non_release_please_versions.sh"
if [ -f "$CHECK_VERSIONS_SCRIPT" ]; then
echo "Updating exclusions in $CHECK_VERSIONS_SCRIPT..."
# Insert the new module name before the .github exclusion
sed -i "/\.github\*\./ i \ [[ \"\${pomFile}\" =~ .*$SOURCE_REPO_NAME.* ]] || \\\\" "$CHECK_VERSIONS_SCRIPT"

echo "Committing $CHECK_VERSIONS_SCRIPT update..."
git add "$CHECK_VERSIONS_SCRIPT"
git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): add to exclusions in $CHECK_VERSIONS_SCRIPT"
COMMIT_COUNT=$((COMMIT_COUNT + 1))
fi

# 7.7 Consolidate versions.txt
Copy link
Member

@suztomo suztomo Mar 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In #12019, I ran ../generation/apply_versions.sh ../versions.txt current in the java-bigquery directory to fix the discrepancies in the Maven module versions. Why don't you run this in the migrated module to have the correct dependency versions?

echo "Consolidating versions.txt..."
SOURCE_VERSIONS="$SOURCE_REPO_NAME/versions.txt"
Expand Down Expand Up @@ -323,10 +487,34 @@ fi
# git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): update copyright headers to 2026 Google LLC"
# COMMIT_COUNT=$((COMMIT_COUNT + 1))

# 7.10 Modernize pom.xml files and extract google-api-services dependencies
modernize_and_extract() {
local pom_file="$1"
shift
local cmd_output
cmd_output=$(python3 "$MODERNIZE_POM_SCRIPT" "$pom_file" "$@" 2>&1)
echo "$cmd_output" | grep -v "EXTRACT_DEP:" || true

# Process extracted dependencies
echo "$cmd_output" | grep "EXTRACT_DEP:" | while read -r line; do
dep_info="${line#EXTRACT_DEP:}"
IFS=':' read -r gid aid ver <<< "$dep_info"
echo "Extracting $gid:$aid:$ver to parent POM..."
python3 "$UPDATE_DEP_MGMT_SCRIPT" "google-cloud-jar-parent/pom.xml" "$gid" "$aid" "$ver"

# Commit parent POM change if anything changed
if git diff --name-only | grep -q "google-cloud-jar-parent/pom.xml"; then
git add "google-cloud-jar-parent/pom.xml"
git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): manage $aid in parent POM"
COMMIT_COUNT=$((COMMIT_COUNT + 1))
fi
done
}

# 7.11 Modernize root pom.xml
echo "Modernizing root pom.xml..."
PARENT_VERSION=$(grep -m 1 "<version>.*{x-version-update:google-cloud-java:current}" google-cloud-jar-parent/pom.xml | sed -E 's/.*<version>(.*)<\/version>.*/\1/')
python3 "$MODERNIZE_POM_SCRIPT" "$SOURCE_REPO_NAME/pom.xml" "$PARENT_VERSION" "$SOURCE_REPO_NAME"
modernize_and_extract "$SOURCE_REPO_NAME/pom.xml" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME"

echo "Committing root pom.xml modernization..."
git add "$SOURCE_REPO_NAME/pom.xml"
Expand All @@ -340,18 +528,38 @@ echo "Modernizing BOM pom.xml..."
while read -r bom_pom; do
echo "Modernizing BOM: $bom_pom"
# BOMs should inherit from google-cloud-pom-parent
python3 "$MODERNIZE_POM_SCRIPT" "$bom_pom" "$PARENT_VERSION" "$SOURCE_REPO_NAME" "google-cloud-pom-parent" "../../google-cloud-pom-parent/pom.xml"
modernize_and_extract "$bom_pom" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" --parent-artifactId "google-cloud-pom-parent" --relative-path "../../google-cloud-pom-parent/pom.xml"

echo "Committing BOM pom.xml modernization for $bom_pom..."
git add "$bom_pom"
git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): modernize BOM pom.xml"
COMMIT_COUNT=$((COMMIT_COUNT + 1))
done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep -v "samples")

# 7.12b Modernize other pom.xml files
echo "Modernizing other pom.xml files..."
while read -r other_pom; do
echo "Modernizing submodule POM: $other_pom"
# Preserve the existing parent, but update everything else
modernize_and_extract "$other_pom" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" --keep-parent

echo "Committing submodule pom.xml modernization for $other_pom..."
git add "$other_pom" && git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): modernize submodule pom.xml" && COMMIT_COUNT=$((COMMIT_COUNT + 1)) || true
done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep -v "\-bom/pom.xml" | grep -v "samples" | grep -v "test_data")

# 7.11 Verify compilation
echo "Verifying compilation..."
BUILD_SUBDIR="${SOURCE_REPO_NAME}" JOB_TYPE=test .kokoro/build.sh
# (cd "$SOURCE_REPO_NAME" && mvn compile -DskipTests -T 1C)
if [[ "${SKIP_TESTS:-false}" != "true" ]]; then
BUILD_SUBDIR="${SOURCE_REPO_NAME}" JOB_TYPE=test .kokoro/build.sh
fi

# 7.12 Apply manual changes
if [[ -f "~/${SOURCE_REPO_NAME}.diff" ]]; then
echo "Applying diff from ${SOURCE_REPO_NAME}.diff..."
git apply "~/${SOURCE_REPO_NAME}.diff"
echo "Committing diff..."
git commit -am "manual changes"
fi

# 7.13 Squash commits
if [ "${SQUASH_COMMITS:-false}" = "true" ]; then
Expand All @@ -370,7 +578,7 @@ if [ "${SQUASH_COMMITS:-false}" = "true" ]; then
# then commit --amend adds those staged changes to C1.

git reset --soft "HEAD~$((COMMIT_COUNT - 1))"
git commit --amend --no-edit --no-gpg-sign
git commit --amend --no-edit --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate $SOURCE_REPO_NAME into monorepo"
echo "Squashed everything into one commit."
fi
fi
Expand Down
Loading
Loading