From 855e8c81e8792780d518b333c1133bb9009094d9 Mon Sep 17 00:00:00 2001 From: BradBot_1 <45739045+BradBot1@users.noreply.github.com> Date: Wed, 22 Feb 2023 22:57:04 +0000 Subject: [PATCH] init --- .dockerignore | 7 +++ .drone.yml | 16 ++++++ .gitignore | 2 + Dockerfile | 24 +++++++++ README.md | 1 + data.json | 24 +++++++++ docker-compose.yml | 16 ++++++ package.json | 17 +++++++ src/Forward/Forward.ts | 24 +++++++++ src/Forward/ForwardManager.ts | 32 ++++++++++++ src/Forward/Recipient.ts | 24 +++++++++ src/Git/GitAuthor.ts | 15 ++++++ src/Git/GitManager.ts | 75 +++++++++++++++++++++++++++++ src/Server/Server.ts | 35 ++++++++++++++ src/Server/Webhook.ts | 12 +++++ src/index.ts | 91 +++++++++++++++++++++++++++++++++++ start.sh | 15 ++++++ tsconfig.json | 15 ++++++ 18 files changed, 445 insertions(+) create mode 100644 .dockerignore create mode 100644 .drone.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 data.json create mode 100644 docker-compose.yml create mode 100644 package.json create mode 100644 src/Forward/Forward.ts create mode 100644 src/Forward/ForwardManager.ts create mode 100644 src/Forward/Recipient.ts create mode 100644 src/Git/GitAuthor.ts create mode 100644 src/Git/GitManager.ts create mode 100644 src/Server/Server.ts create mode 100644 src/Server/Webhook.ts create mode 100644 src/index.ts create mode 100644 start.sh create mode 100644 tsconfig.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fb57bd3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +node_modules/ +temp/ +package-lock.json +Dockerfile +docker-compose.yml +data.json +.gitignore \ No newline at end of file diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..399cbc5 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,16 @@ +kind: pipeline +name: default + +steps: +- name: docker + image: plugins/docker + settings: + username: bradbot1 + password: + from_secret: access_token + repo: bradbot1/gitea-forwarder + tags: latest + +trigger: + branch: + - master \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ccb2c80 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..82d714f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM node:alpine + +RUN apk update +RUN apk add git +RUN apk add doas +RUN apk add --update --no-cache openssh-client + +RUN mkdir -p -m 0700 ~/.ssh +RUN ssh-keyscan github.com >> ~/.ssh/known_hosts +RUN ssh-keyscan git.bb1.fun >> ~/.ssh/known_hosts +RUN ssh-keyscan gitlab.com >> ~/.ssh/known_hosts +RUN ssh-keyscan bitbucket.org >> ~/.ssh/known_hosts +RUN ssh-keyscan codeberg.org >> ~/.ssh/known_hosts +#RUN eval `ssh-agent -s` +WORKDIR /usr/app + +COPY ./ ./ + +RUN npm install +RUN npm run build +RUN rm ./src/ -fr +RUN rm tsconfig.json + +ENTRYPOINT [ "./start.sh" ] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..fd2db40 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# Gitea Forwarder \ No newline at end of file diff --git a/data.json b/data.json new file mode 100644 index 0000000..91908cb --- /dev/null +++ b/data.json @@ -0,0 +1,24 @@ +[ + { + "origin": "https://git.example.com/user/repo", + "webhook": "CustomWebhookSlug", + "recipients": [ + { + "url": "https://gitlab.com/user/repo", + "humanName": "An optional human name for logging", + "authors": [ + { + "old": "user@example.com", + "email": "00000000+user@users.noreply.github.com", + "name": "user" + }, + { + "old": "user2@example.com", + "email": "00000001+user2@users.noreply.github.com", + "name": "user2" + } + ] + } + ] + } +] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..255798a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3' + +services: + forwarder: + build: . + image: bradbot1/gitea-forwarder + ports: + - '80:80' + environment: + PORT: 80 + # if you want ssl + #SSL_CERT_PATH: + #SSL_KEY_PATH: + #SSL_PASSWORD: + volumes: + - ./data.json:./data.json \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..715ef12 --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "scripts": { + "dev": "ts-node-dev --clear ./src/index.ts", + "build": "tsc", + "start": "node dist/index.js" + }, + "dependencies": { + "0http": "^3.4.3", + "body-parser": "^1.20.1", + "low-http-server": "^4.1.0", + "simple-git": "^3.16.1" + }, + "devDependencies": { + "ts-node-dev": "^2.0.0", + "typescript": "^4.9.5" + } +} diff --git a/src/Forward/Forward.ts b/src/Forward/Forward.ts new file mode 100644 index 0000000..d4bb6ab --- /dev/null +++ b/src/Forward/Forward.ts @@ -0,0 +1,24 @@ +import { Recipient } from './Recipient' + +export class Forward { + + public readonly webhook: string; + public readonly origin: string; + private readonly recipients: Recipient[] = []; + + constructor(webhookId: string, origin: string) { + this.webhook = webhookId; + this.origin = origin; + } + + public createRecipient(url: string, humanName: string|undefined):Recipient { + const recipient = new Recipient(url, humanName); + this.recipients.push(recipient); + return recipient; + } + + public getRecipients(): Recipient[] { + return this.recipients; + } + +} \ No newline at end of file diff --git a/src/Forward/ForwardManager.ts b/src/Forward/ForwardManager.ts new file mode 100644 index 0000000..26c189e --- /dev/null +++ b/src/Forward/ForwardManager.ts @@ -0,0 +1,32 @@ +import { Forward } from './Forward'; + +const known_forwards: Forward[] = []; +/** + * Generates a 32 character long random string +*/ +function generateRandomWebhookId(): string { + return (Math.random().toString(36).substring(2) + +Math.random().toString(36).substring(2) + +Math.random().toString(36).substring(2) + +Math.random().toString(36).substring(2, 2)); +} + +export function getForwardByWebhook(webhookId: string): Forward|null { + for (var forward of known_forwards) + if (forward.webhook === webhookId) + return forward; + return null; +}; + +export function getForwards(): Forward[] { + return known_forwards; +}; + +export function createForward(origin: string, webhook: string|undefined): Forward { + if (webhook === undefined) { + do { + webhook = generateRandomWebhookId(); + } while (getForwardByWebhook(webhook) != null); + } + return new Forward(webhook, origin); +}; \ No newline at end of file diff --git a/src/Forward/Recipient.ts b/src/Forward/Recipient.ts new file mode 100644 index 0000000..858a8c3 --- /dev/null +++ b/src/Forward/Recipient.ts @@ -0,0 +1,24 @@ +import { GitAuthor } from '../Git/GitAuthor'; + +export class Recipient { + + public readonly humanName: string; + public readonly url:string; + public authorMap: Map = new Map(); + + constructor(url: string, humanName: string|undefined) { + this.url = url; + this.humanName = humanName||url; + } + + public setAuthor(gitAuthor: string, newName: string, newEmail: string):void { + this.authorMap.set(gitAuthor, new GitAuthor(newName, newEmail)); + } + + public formatAuthor(gitAuthor: string): string { + const author: GitAuthor|undefined = this.authorMap.get(gitAuthor); + if (author === undefined) return gitAuthor; + return author.format(); + } + +} \ No newline at end of file diff --git a/src/Git/GitAuthor.ts b/src/Git/GitAuthor.ts new file mode 100644 index 0000000..78d9881 --- /dev/null +++ b/src/Git/GitAuthor.ts @@ -0,0 +1,15 @@ +export class GitAuthor { + + public readonly name: string; + public readonly email: string; + + constructor(name: string, email: string) { + this.name = name; + this.email = email; + } + + public format(): string { + return `${this.name} <${this.email}>`; + } + +} \ No newline at end of file diff --git a/src/Git/GitManager.ts b/src/Git/GitManager.ts new file mode 100644 index 0000000..b9fb617 --- /dev/null +++ b/src/Git/GitManager.ts @@ -0,0 +1,75 @@ +import { Forward } from '../Forward/Forward' +import { simpleGit as Git } from 'simple-git'; +import { GitAuthor } from './GitAuthor'; +import { Recipient } from '../Forward/Recipient'; +import { execSync } from 'child_process'; +import { writeFileSync } from 'fs' + +export async function cloneRepo(repo: string, out: string = __dirname): Promise { + try { + await Git().clone(repo, out); + } catch (error) { + console.log(error); + await Git(out).pull(); + } +} + +export async function push(repo: string, out: string = __dirname): Promise { + const git = Git(out); + git.addRemote("fumo", repo); + git.push("fumo"); +} + +export async function getCommitAuthors(out: string = __dirname): Promise { + const git = Git(out); + const oldAuthors: Set = new Set(); + for (const commit of (await git.log()).all) + oldAuthors.add(commit.author_email); + return Array.from(oldAuthors); +} + +const _changeCommitAuthorsReplace: string = `#!/bin/sh + +git filter-branch --env-filter ' + +an="$GIT_AUTHOR_NAME" +am="$GIT_AUTHOR_EMAIL" +cn="$GIT_COMMITTER_NAME" +cm="$GIT_COMMITTER_EMAIL" + +echo $GIT_COMMITTER_EMAIL + +if [ "$GIT_COMMITTER_EMAIL" = "OLD_EMAIL" ] +then + cn="NEW_NAME" + cm="NEW_EMAIL" +fi +if [ "$GIT_AUTHOR_EMAIL" = "OLD_EMAIL" ] +then + an="NEW_NAME" + am="NEW_EMAIL" +fi + +export GIT_AUTHOR_NAME="$an" +export GIT_AUTHOR_EMAIL="$am" +export GIT_COMMITTER_NAME="$cn" +export GIT_COMMITTER_EMAIL="$cm" +'`; + + +export async function changeCommitAuthors(recipient: Recipient, out: string = __dirname): Promise { + for (const author of await getCommitAuthors(out)) { + const newAuthor: GitAuthor|undefined = recipient.authorMap.get(author); + if (newAuthor === undefined) continue; + writeFileSync(out + "git_change.sh", _changeCommitAuthorsReplace.replace(/OLD_EMAIL/g, author).replace(/NEW_NAME/g, newAuthor.name).replace(/NEW_EMAIL/g, newAuthor.email)); + execSync("/bin/sh " + out + "git_change.sh", { + cwd: out + }); + } +} + +export async function handleForward(forward: Forward):Promise { + const git = Git().env('GIT_SSH_COMMAND', 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'); + await git.clone(forward.origin); + git.log() +} \ No newline at end of file diff --git a/src/Server/Server.ts b/src/Server/Server.ts new file mode 100644 index 0000000..eda2855 --- /dev/null +++ b/src/Server/Server.ts @@ -0,0 +1,35 @@ +import { setupWebhookRoutes, WebhookRecievedCallback } from './Webhook'; + +export function createServer(port: number, webhookRecievedCallback: WebhookRecievedCallback): void { + const { router, server } = require('0http')({ + defaultRoute: (_: any, res: any) => { + res.statusCode = 404; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end('{"error":"route_not_found",message:"There route requested does not exist!"}'); + }, + // router: require('find-my-way')(), + server: require('low-http-server')({ + cert_file_name: process.env["SSL_CERT_PATH"], + key_file_name: process.env["SSL_KEY_PATH"], + password: process.env["SSL_PASSWORD"] + }) + }); + + router.use(require('body-parser').json({ + limit: '5kb', + strict: true, + type: "application/json" + })); + + router.get('/', (_:any, res:any) => { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end('{"name":"GiteaForwarder","version":"1.0.0","about":"Forwarding git data to various repositories!"}'); + }) + + setupWebhookRoutes(router, webhookRecievedCallback); + + server.listen(port, () => { + console.log("[WEB] Now listening on port " + port); + }); +} \ No newline at end of file diff --git a/src/Server/Webhook.ts b/src/Server/Webhook.ts new file mode 100644 index 0000000..dba4da8 --- /dev/null +++ b/src/Server/Webhook.ts @@ -0,0 +1,12 @@ +export type WebhookRecievedCallback = (webhookId: string, webhook: Promise) => void; + +export function setupWebhookRoutes(router:any, webhookRecievedCallback:WebhookRecievedCallback):void { + + router.post('/webhook/:webhook', (req: any, res: any) => { + const webhookId = req.params.webhook; + console.log("[WEB] Recieved webhook " + webhookId); + webhookRecievedCallback(webhookId, res.body); + res.statusCode = 200; + res.end(); + }); +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..e01d347 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,91 @@ +import { Forward } from './Forward/Forward'; +import { createForward, getForwardByWebhook } from './Forward/ForwardManager'; +import { createServer } from './Server/Server'; +import { env, cwd } from 'process'; +import { cloneRepo, changeCommitAuthors, push } from './Git/GitManager'; +import { rmSync, existsSync, readFileSync } from 'fs' + +var dataToLoad: string; + +if (env.hasOwnProperty("DATA")) { + dataToLoad = env["DATA"]||""; +} else if (existsSync("./data.json")) { + dataToLoad = readFileSync("./data.json").toString(); +} else { + console.error("Failed to find any data to load!"); + process.exit(1); +} + +var parsedData: any; + +try { + parsedData = JSON.parse(dataToLoad); +} catch (e) { + console.error("Failed to parse the provided data!"); + process.exit(1); +} + +if (!Array.isArray(parsedData)) parsedData = [parsedData]; + +for (const forwardData of parsedData) { + if (!forwardData.hasOwnProperty("origin")) { + console.error("No origin of forward!"); + continue; + } + const forward = createForward(forwardData.origin, forwardData.webhook); + console.log("Created forward on webhook: " + forward.webhook); + if (forwardData.hasOwnProperty("recipients")) { + for (const recipientData of forwardData.recipients) { + if (!recipientData.hasOwnProperty("url")) { + console.error("No url provided for recipient"); + continue; + } + const recipient = forward.createRecipient(recipientData.url, recipientData.humanName); + if (!recipientData.hasOwnProperty("authors")) { + console.error("No authors provided for recipient: " + recipient.humanName||recipient.url); + continue; + } + for (const author of recipientData.authors) { + if (!author.hasOwnProperty("old")) { + console.error("No old email provided for recipient: " + author.humanName||recipient.url); + continue; + } + if (!author.hasOwnProperty("email")) { + console.error("No new email provided for recipient: " + author.humanName||recipient.url); + continue; + } + if (!author.hasOwnProperty("name")) { + console.error("No new name provided for recipient: " + author.humanName||recipient.url); + continue; + } + recipient.setAuthor(author.old, author.name, author.email); + } + } + } else { + console.log("No recipients found"); + } +} + +createServer(parseInt(env["PORT"]||"3000"), async (webhookId: string, webhook: Promise) => { + const forward: Forward|null = getForwardByWebhook(webhookId); + if (forward == null) return; + const webhookData: any = await webhook; + if (!webhookData.hasOwnProperty("repository")) return; + const url: any = webhookData.repository.clone_url || webhookData.repository.ssh_url || webhookData.repository.html_url; + if (typeof url !== "string") return; + if (forward.origin !== url) { + console.log("[VAL] Invalid webhook origin!"); + return; + } + console.log("[VAL] Validated " + url); + const outputDir = cwd() + "/git_output_" + Math.random().toString(32).substring(2,8) + "/"; + for (const sendTo of forward.getRecipients()) { + await cloneRepo(forward.origin, outputDir); + await changeCommitAuthors(sendTo, outputDir); + await push(sendTo.url, outputDir); + rmSync(outputDir, { + recursive: true, + force: true + }); + } +}); \ No newline at end of file diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..0cd4f55 --- /dev/null +++ b/start.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +if ! [ -f "/root/.ssh/id_rsa.pub" ]; then + echo "No key found, creating a new one!" + ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -P "" + chmod 700 /~/.ssh/id_rsa + chmod 700 /~/.ssh/id_rsa.pub + echo " IdentityFile ~/.ssh/id_rsa" >> /etc/ssh/ssh_config + echo "Your new public key for this instance:" + cat ~/.ssh/id_rsa.pub +fi + +npm run start + +/bin/sh \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..db126cf --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es2016", + "module": "commonjs", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "rootDir": "src", + "outDir": "dist", + "noImplicitAny": true, + "allowJs": true + }, + "exclude": [ "dist", "temp" ] +}