Some code that moves things around between git repositories
I used this to refactor a large java app recently. You may be able to pick the bones out of it somehow
#!/bin/bash
configjson=$(cat <<-END
{
"casper": {
"repourl":"https://mobius-gitlab.bt.com/dsl/myee/casper.git",
"packagename":"uk.co.ee.myee",
"basedir": "/Users/richard/scripts/java/casper",
"core": []
},
"casper-core": {
"repourl":"https://mobius-gitlab.bt.com/dsl/myee/original/casper-core.git",
"packagename":"uk.co.ee.myee.core",
"basedir": "/Users/richard/scripts/java/casper-core",
"core": []
},
"casper-tags": {
"repourl":"https://mobius-gitlab.bt.com/dsl/myee/original/casper-tags.git",
"packagename":"uk.co.ee.myee.tags",
"basedir": "/Users/richard/scripts/java/casper-tags",
"core": [
"src/main/java/uk/co/ee/myee/tags/analytic/",
"src/main/java/uk/co/ee/myee/tags/Functions.java",
"src/main/java/uk/co/ee/myee/tags/enums/Environment.java",
"src/main/java/uk/co/ee/myee/tags/cms/CMSType.java",
"src/main/java/uk/co/ee/myee/tags/cms/MyEEAEMConfig.java",
"src/main/java/uk/co/ee/myee/tags/cms/styleguide",
"src/main/java/uk/co/ee/myee/tags/forms/FormTag.java",
"src/main/java/uk/co/ee/myee/tags/forms/GroupTagWriter.java",
"src/main/java/uk/co/ee/myee/tags/forms/TagContext.java",
"src/main/java/uk/co/ee/myee/tags/html/AbstractPageElement.java",
"src/main/java/uk/co/ee/myee/tags/model/",
"src/main/java/uk/co/ee/myee/tags/util/"
]
},
"casper-service": {
"repourl":"https://mobius-gitlab.bt.com/dsl/myee/original/casper-service.git",
"packagename":"uk.co.ee.myee.service",
"basedir": "/Users/richard/scripts/java/casper-service",
"core": [
"src/main/java/uk/co/ee/myee/service/client/security/",
"src/main/java/uk/co/ee/myee/service/analytic/AnalyticDataAggregator.java",
"src/main/java/uk/co/ee/myee/service/analytic/AnalyticDataKeys.java",
"src/main/java/uk/co/ee/myee/service/analytic/TealiumConfig.java",
"src/main/java/uk/co/ee/myee/service/analytic/impl/UpgradeDetails.java",
"src/main/java/uk/co/ee/myee/service/captcha/utils/CaptchaConstants.java",
"src/main/java/uk/co/ee/myee/service/domain/recommendations/Recommendation.java",
"src/main/java/uk/co/ee/myee/service/facade/billingaccount/domain/DeviceAndContractResponse.java",
"src/main/java/uk/co/ee/myee/service/facade/billingaccount/BillingAccountFacade.java",
"src/main/java/uk/co/ee/myee/service/delegate/makepayment/dto/cancel/request/BuynetPaymentRequestMessage.java",
"src/main/java/uk/co/ee/myee/service/delegate/makepayment/dto/cancel/response.BuynetPaymentResponseMessage.java",
"src/main/java/uk/co/ee/myee/service/delegate/makepayment/dto/request/BuynetPaymentEnquiryRequestMessage.java",
"src/main/java/uk/co/ee/myee/service/delegate/makepayment/dto/response/BuynetPaymentEnquiryResponseMessage.java",
"src/main/java/uk/co/ee/myee/service/upgradehub/UpgradeHubService.java",
"src/main/java/uk/co/ee/myee/service/upgradehub/dto/AnytimeUpgradeItem.java",
"src/main/java/uk/co/ee/myee/service/upgradehub/dto/UpgradeItem.java",
"src/main/java/uk/co/ee/myee/service/upgradehub/dto/UpgradeType.java",
"src/main/java/uk/co/ee/myee/service/delegate/allowancesprepay/AllowancesPrepayDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/allowancesprepay/JsonFormatException.java",
"src/main/java/uk/co/ee/myee/service/delegate/benefits/dto/BenefitsDetails.java",
"src/main/java/uk/co/ee/myee/service/delegate/benefits/dto/BenefitsUpdate.java",
"src/main/java/uk/co/ee/myee/service/delegate/billingaccount/impl/AccountPaymentStatus.java",
"src/main/java/uk/co/ee/myee/service/delegate/context/ContextDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/documents/DocumentsDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/identity/IdentityDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/makepayment/dto/enquire/request/BuynetPaymentEnquiryRequestMessage.java",
"src/main/java/uk/co/ee/myee/service/delegate/makepayment/dto/enquire/response/BuynetPaymentEnquiryResponseMessage.java",
"src/main/java/uk/co/ee/myee/service/delegate/network/NetworkDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/onetimepin/dto/AuthClassReference.java",
"src/main/java/uk/co/ee/myee/service/delegate/pack/PackDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/saleseligibility/SalesEligibilityDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/subscription/SubscriptionDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/subscription/contract/ContractDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/subscription/contract/DeviceDetails.java",
"src/main/java/uk/co/ee/myee/service/delegate/subscription/contract/PlanDetails.java",
"src/main/java/uk/co/ee/myee/service/directpurchase/domain/ProductSessionData.java",
"src/main/java/uk/co/ee/myee/service/facade/billingaccount/BillingAccountFacade.java",
"src/main/java/uk/co/ee/myee/service/managedevice/dto/SpendLimitData.java",
"src/main/java/uk/co/ee/myee/service/paymentcard/domain/PaymentCardDetails.java",
"src/main/java/uk/co/ee/myee/service/delegate/BaseDelegate.java",
"src/main/java/uk/co/ee/myee/service/delegate/billingaccount/impl/LoanAccountStatus.java",
"src/main/java/uk/co/ee/myee/service/delegate/onetimepin/dto/AuthResponse.java",
"src/main/java/uk/co/ee/myee/service/delegate/pack/dto/AddPackRequest.java",
"src/main/java/uk/co/ee/myee/service/delegate/pack/dto/UpdatePackRequest.java",
"src/main/java/uk/co/ee/myee/service/delegate/ThreeDSV2Authentication.java",
"src/main/java/uk/co/ee/myee/service/delegate/result/ErrorAuth.java"
]
},
"casper-flow": {
"repourl":"https://mobius-gitlab.bt.com/dsl/myee/original/casper-flow.git",
"packagename":"uk.co.ee.myee.flow",
"basedir": "/Users/richard/scripts/java/casper-flow",
"core": [
"src/main/java/uk/co/ee/myee/flow/base/",
"src/main/java/uk/co/ee/myee/flow/errors/",
"src/main/java/uk/co/ee/myee/flow/viewbill/optimization/constants/PlanBillApiConstants.java",
"src/main/java/uk/co/ee/myee/flow/viewbill/optimization/constants/PlanBillConstants.java"
]
}
}
END
)
packageMappings=(
'uk.co.ee.myee.service.client.base:uk.co.ee.myee.core.client.base'
'javax.servlet:jakarta.servlet'
'javax.el:jakarta.el'
'javax.xml:jakarta.xml'
'javax.annotation:jakarta.annotation'
)
function getAdminPassword {
if [ -z "$ADMIN_PASSWORD" ]; then
echo "Before running this script, you must export your laptop admin password to ADMIN_PASSWORD"
exit 1
fi
echo "$ADMIN_PASSWORD"
}
function conf {
if [ -z "$1" ]; then
echo "must supply (in first param) the module name (casper, casper-flow, casper-core etc.)"
exit 1
fi
if [ -z "$2" ]; then
echo "must supply the name of the required variable in the second param (ie. sourcerepo etc.)"
exit 1
fi
local m="$(echo "$configjson" | jq .\"$1\")"
local v="$(echo "$m" | jq -r .\"$2\")"
echo "$v"
}
function doSpecificReverts {
cd $targetbase
git checkout HEAD -- src/main/java/uk/co/ee/myee/core/auth/AuthDelegate.java
git checkout HEAD -- src/main/java/uk/co/ee/myee/core/auth/AuthDelegateImpl.java
git checkout HEAD -- src/main/java/uk/co/ee/myee/core/auth/AuthenticationImpl.java
}
# parse the source directory looking for files that start with 'import uk.co.ee.myee'
# returns anything that doesn't also contain the real package name (ie uk.co.ee.core)
function getAlienImports {
if [ -z "$1" ]; then
echo "must supply package name in first param (ie casper-core etc.)"
exit 1
fi
local dir="$(conf "$1" 'basedir')"
cd $dir
local p="$(conf "$1" 'packagename')"
echo "checking in $dir for any package which is not $p"
grep -rwh --exclude="*.txt" "$dir" -e "import uk.co.ee.myee" | grep -v "$p" | sort -u
grep -rwh --exclude="*.txt" "$dir" -e "import javax" | sort -u
}
function replaceAll {
if [ -z "$1" ]; then
echo "must supply target string in first parameter"
exit 1
fi
if [ -z "$2" ]; then
echo "must supply replacement string in second parameter"
exit 1
fi
local OS=`uname`
local ef="$1"
local er="$2"
#local ef="$(echo "$2" | sed -e 's/[]\/$*.^[]/\\&/g')"
#local er="$(echo "$3" | sed -e 's/[]\/$*.^[]/\\&/g')"
if [ "$OS" = 'Darwin' ]; then
find . -name '*.java' -exec sed -i '' "s/$ef/$er/g" {} \;
else
find . -name '*.java' -exec sed -i "s/$ef/$er/g" {} \;
fi
}
function recloneRepo {
if [ -z "$1" ]; then
echo "must supply module name in the first parameter (ie casper-core)"
exit 1
fi
# currently requires that these directories already exist
local sourcebase="$(conf "$1" 'basedir')"
local sourcerepo="$(conf "$1" 'repourl')"
if [ ! -d $sourcebase ]; then
echo "temporarily creating $sourcebase"
mkdir -p $sourcebase
fi
cd $sourcebase
cd ../
local ap="$(getAdminPassword)"
echo "deleting $sourcebase"
# echo "$ap" | sudo -S rm -R $sourcebase 2> /dev/null
echo "$ap" | sudo -S rm -R $sourcebase
echo "cloning $sourcebase"
git clone $sourcerepo
}
function getRepoName {
if [ -z "$1" ]; then
echo "must supply the repo base dir absolute path in the first parameter"
exit 1
fi
if [ -d "$1" ]; then
cd $1
local rn="$(git config --get remote.origin.url)"
if [ -z "${rn}" ]; then
echo "failed to get the remote origin url $(pwd)"
exit 1;
fi
local bn=$(basename -s .git $rn)
echo "$bn"
else
local m=`echo "$configjson" | jq -r "to_entries[] | select(.value.basedir == \"$1\") | .key"`
echo "$m"
fi
}
function revertSourceToMain {
cd $sourcebase
echo "Hard resetting $(getRepoName $sourcebase) to origin/main"
git fetch origin
git fetch --all
git reset --hard origin/main
git checkout main
git pull
}
# Takes something like uk.co.ee.myee.core and returns something like
# src/main/java/uk/co/ee/myee/core
function getSubpathFromPackagename {
if [ -z "$1" ]; then
echo "must supply target string (ie uk.co.ee.myee.core) in the first param"
exit 1
fi
local ret="src/main/java/${1//.//}"
echo "$ret"
}
# Takes something like src/main/java/uk/co/ee/myee/tags/Functions.java and returns something like
# uk.co.ee.myee.tags.Functions
function getPackagenameFromSubPath {
if [ -z "$1" ]; then
echo "must supply target string (ie uk/co/ee/myee/core) in the first param"
exit 1
fi
local ret="${1%'.java'}"
local ret="${ret%'.*'}"
local ret="${ret////.}"
local ret="${ret#'src.main.java.'}"
echo "$ret"
}
# returns the target package appended with the difference between it and the source package
# ie if source is uk.co.ee.myee.tags.foobar and target is uk.co.ee.myee.core
# the result will be src/main/java/uk/co/ee/myee/core/tags/foobar
function calculateNewPathName {
if [ -z "$1" ]; then
echo "must supply target package in first param (ie casper-core)"
exit 1
fi
if [ -z "$2" ]; then
echo "must supply filepath you wish to convert in second param"
exit 1
fi
local targetpackage="$(conf "$1" 'packagename')"
# local tp="src/main/java/${targetpackage//.//}"
local tp=$(getSubpathFromPackagename "$targetpackage")
local foo=$(printf "%s\n%s\n" "$tp" "$2" | sed -e 'N;s/^\(.*\).*\n\1.*$/\1/')
local foo=${2#"$foo"}
local ret="$tp/$foo"
echo "$ret"
}
# returns the target package appended with the difference between it and the source package
# ie if source is uk.co.ee.myee.tags.foobar and target is uk.co.ee.myee.core
# the result will be uk.co.ee.myee.core.tags.foobar
function calculateNewPackageName {
if [ -z "$1" ]; then
echo "must supply from package in first param (ie casper-tags)"
exit 1
fi
if [ -z "$2" ]; then
echo "must supply to package in first param (ie casper-core)"
exit 1
fi
local sourcepackage="$(conf "$1" 'packagename')"
local targetpackage="$(conf "$2" 'packagename')"
local foo=$(printf "%s\n%s\n" "$sourcepackage" "$targetpackage" | sed -e 'N;s/^\(.*\).*\n\1.*$/\1/')
local foo=${sourcepackage#"$foo"}
local ret="$targetpackage.$foo"
echo "$ret"
}
# use this for extracting large directories from one repo and placing them
# into another, rather than using the splitSingleFiles function which would work
# but would not be as clean
function splitDirectory {
local branchname="sfi-branch"
if [ -z "$1" ]; then
echo "must supply from package in first param (ie casper-tags)"
exit 1
fi
if [ -z "$2" ]; then
echo "must supply to package in first param (ie casper-core)"
exit 1
fi
if [ -z "$3" ]; then
echo "must supply relative path (or space delimited list of paths) of the file/s to split from the root directory of the 'from' module (ie src/main/java/uk/co/ee/myee/tags/Functions.java"
exit 1
fi
recloneRepo "$1"
recloneRepo "$2"
local sourcebase="$(conf "$1" 'basedir')"
local targetbase="$(conf "$2" 'basedir')"
local targetpath="$(calculateNewPathName "$2" $3)"
echo "splitting out $3 from $1 into branch named $branchname"
cd $sourcebase
echo "splitting $3 from $sourcebase and placing it into $targetbase at $targetpath"
git subtree split -P $3 -b $branchname
git checkout $branchname
local rr="$(getRepoName $sourcebase)"
echo "copying branch from $1 into $2 with same branch name"
cd $targetbase
git remote add -f $rr $sourcebase
git merge --no-commit --allow-unrelated-histories $rr/$branchname
git checkout $branchname
echo "moving copied files into suitable location [$targetpath]"
mkdir -p "./$targetpath"
git mv --force ./* ./$targetpath -k
git add .
git commit -m "moved into src package"
git checkout main
echo "Merging into main"
git merge $branchname --no-commit --allow-unrelated-histories
git remote remove $rr
}
function replaceAllMovedPackageDeclarations {
local k=$(echo "$configjson" | jq -r 'keys[]')
local keys=($k)
for basename in "${keys[@]}"; do
local base="$(conf "$basename" 'basedir')"
cd $base
for key in "${keys[@]}"; do
local f=`echo "$configjson" | jq -r ".\"$key\".core[]"`
local files=($f)
for i in "${files[@]}"; do
local newpath="$(calculateNewPathName "casper-core" $i)"
local newdir="$(dirname ${newpath})"
# amend package statement to reflect new location
local op="import `getPackagenameFromSubPath "$i"`"
local np="import `getPackagenameFromSubPath "$newpath"`"
echo "replacing $op for $np in all source files of $basename"
replaceAll "$op" "$np"
done
done
done
}
# copies a some local repo into another, with history
# can accept an array of single files
# requires git-filter-repo (ie python -m pip install --user git-filter-repo)
# TODO look at using native git filter-branch command instead of custom filter-repo module ?
function splitSingleFiles {
local branchname="sfi-branch"
if [ -z "$1" ]; then
echo "must supply from package in first param (ie casper-tags)"
exit 1
fi
if [ -z "$2" ]; then
echo "must supply to package in first param (ie casper-core)"
exit 1
fi
# get some config values
local newpackagename="$(calculateNewPackageName $1 $2)"
local sourcepackage="$(conf "$1" 'packagename')"
local targetrepourl="$(conf "$2" 'repourl')"
local targetpackage="$(conf "$2" 'packagename')"
local sourcebase="$(conf "$1" 'basedir')"
local targetbase="$(conf "$2" 'basedir')"
local sourcereponame="$(getRepoName $sourcebase)"
local targetreponame="$(getRepoName $targetbase)"
local origdir="$targetbase/$(getSubpathFromPackagename $sourcepackage)"
local f=`echo "$configjson" | jq -r ".\"$sourcereponame\".core[]"`
local files=($f)
cd $sourcebase
# create a new branch to work in
git checkout -b "$branchname"
# filter out all the files we don't want
local filter="git filter-repo "
for i in "${files[@]}"; do
local filter+="--path $i "
done
local filter+="--refs refs/heads/$branchname --force"
local foo=$($filter)
cd $targetbase
git checkout -b "$branchname"
git remote add -f $sourcereponame $sourcebase
git fetch $sourcereponame
git branch $targetreponame remotes/$sourcereponame/$branchname
git merge $targetreponame --no-commit --allow-unrelated-histories
# now move files to new package etc.
for i in "${files[@]}"; do
cd $targetbase
local newpath="$(calculateNewPathName $2 $i)"
local newdir="$(dirname ${newpath})"
mkdir -p ./$newdir
git mv --force ./$i ./$newpath -k
cd $sourcebase
git rm -f -r "$i"
done
cd $targetbase
echo "renaming packages in source files to match their new location"
local newpackagename="${newpackagename%'.'}"
replaceAll "package $sourcepackage" "package $newpackagename"
echo "removing stray dir from import : $origdir"
rm -R $origdir
# do commit on both repos if changed
# git commit -m "refactored files from an external repo into local package structure"
# now change all the import statments in all
}
function touchNewFiles {
cd $targetbase
touch ./pom.xml
touch ./.gitignore
touch ./settings.xml
touch ./.gitlab-ci.yml
}
#splitDirectory "casper" "casper-flow" "src/main/java/uk/co/ee/myee/flow"
#getAlienImports "casper-core"
#exit 0
#recloneRepo "casper-service"
recloneRepo "casper-core"
#recloneRepo "casper-tags"
recloneRepo "casper-flow"
splitSingleFiles "casper-flow" "casper-core"
replaceAllMovedPackageDeclarations