Reworked build

2022-12-04 15:50:37 +01:00 · 2022-12-04 15:50:37 +01:00 · 7b92385a7d
parent 93d8ed8d94
commit 7b92385a7d
8 changed files with 364 additions and 48 deletions
--- a/.gitignore
+++ b/.gitignore
@ -55,6 +55,7 @@ cpu.out

 /bin
 /dist
+/docker/bin
 /custom/*
 !/custom/conf
 /custom/conf/*
--- a/.woodpecker/compliance-and-tests.yaml
+++ b/.woodpecker/compliance-and-tests.yaml
@ -1,13 +1,49 @@
 platform: linux/amd64

-depends_on:
- compliance
-
 workspace:
  base: /go
  path: src/codeberg/gitea

 pipeline:
+  deps-backend:
+    image: golang:1.19
+    pull: true
+    secrets:
+      - goproxy_override
+    commands:
+    - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
+    - make deps-backend
+
+  security-check:
+    image: golang:1.19
+    pull: true
+    secrets:
+      - goproxy_override
+    commands:
+    - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
+    - make security-check
+
+  lint-backend:
+    image: gitea/test_env:linux-amd64
+    pull: true
+    secrets:
+      - goproxy_override
+    environment:
+    - TAGS=bindata sqlite sqlite_unlock_notify
+    - GOSUMDB=sum.golang.org
+    commands:
+    - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
+    - make lint-backend
+
+  checks-backend:
+    image: golang:1.19
+    pull: true
+    secrets:
+      - goproxy_override
+    commands:
+    - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
+    - make --always-make checks-backend
+
  fetch-tags:
    image: docker:git
    pull: true
@ -15,12 +51,6 @@ pipeline:
    - git config --add safe.directory '*'
    - git fetch --tags --force

-  deps-backend:
-    image: golang:1.19
-    pull: true
-    commands:
-    - make deps-backend
-
  tag-pre-condition:
    image: drone/git
    pull: true
@ -30,26 +60,36 @@ pipeline:
  prepare-test-env:
    image: gitea/test_env:linux-amd64
    pull: true
+    secrets:
+      - goproxy_override
    commands:
+    - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
    - ./build/test-env-prepare.sh

-  build:
+  build-test:
    image: gitea/test_env:linux-amd64
+    pull: true
+    secrets:
+      - goproxy_override
    environment:
    - GOSUMDB=sum.golang.org
    - TAGS=bindata sqlite sqlite_unlock_notify
    commands:
+    - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
    - su gitea -c './build/test-env-check.sh'
    - su gitea -c 'make backend'

  unit-test:
    image: gitea/test_env:linux-amd64
+    pull: true
+    secrets:
+      - github_read_token
+      - goproxy_override
    environment:
    - TAGS=bindata sqlite sqlite_unlock_notify
    - RACE_ENABLED=true
-    secrets:
-    - github_read_token
    commands:
+    - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
    - su gitea -c 'make unit-test-coverage test-check'

  test-sqlite:
--- a/.woodpecker/compliance.yml
+++ b/.woodpecker/compliance.yml
@ -1,32 +0,0 @@
-platform: linux/amd64
-
-workspace:
-  base: /go
-  path: src/codeberg/gitea
-
-pipeline:
-  deps-backend:
-    image: golang:1.19
-    pull: true
-    commands:
-    - make deps-backend
-
-  security-check:
-    image: golang:1.19
-    pull: true
-    commands:
-    - make security-check
-
-  lint-backend:
-    image: gitea/test_env:linux-amd64
-    pull: true
-    environment:
-    - TAGS=bindata sqlite sqlite_unlock_notify
-    - GOSUMDB=sum.golang.org
-    commands:
-    - make lint-backend
-
-  checks-backend:
-    image: golang:1.19
-    commands:
-    - make --always-make checks-backend
--- a/.woodpecker/release-version.yml
+++ b/.woodpecker/release-version.yml
@ -1,7 +1,7 @@
 platform: linux/amd64

 depends_on:
- testing-amd64
+- compliance-and-testing

 workspace:
  base: /source
@ -19,24 +19,77 @@ pipeline:
    image: node:18
    pull: true
    commands:
+      - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
      - make deps-frontend
+    secrets:
+      - goproxy_override

  deps-backend:
    image: golang:1.19
    pull: true
    commands:
-     - make deps-backend
+      - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
+      - make deps-backend
+    secrets:
+      - goproxy_override

-  static:
+  build-static:
    image: techknowlogick/xgo:go-1.19.x
    pull: true
    commands:
+      - test "$${GOPROXY_OVERRIDE-}" != "" && export GOPROXY="$${GOPROXY_OVERRIDE}"
+      - export WS_BASE="$(pwd)"
+      - echo "installing nodejs"
      - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs
+      - echo "fetching upx"
+      - cd /usr/local/bin
+      - curl -sL https://github.com/upx/upx/releases/download/v$${UPXVERSION}/upx-$${UPXVERSION}-amd64_linux.tar.xz | tar --strip-components 1 -xJf - upx-$${UPXVERSION}-amd64_linux/upx
+      - echo "checking upx"
+      - test -f /usr/local/bin/upx || exit 1
      - export PATH=$PATH:$GOPATH/bin
-      - make CI=true LINUX_ARCHS=linux/amd64,linux/arm64 release
+      - echo "building forgejo"
+      - cd $WS_BASE
+      - make CI=true LINUX_ARCHS=$${LINUX_ARCHS} release
+      - echo "building environment-to-ini"
+      - export TAGS=bindata
+      - cd $WS_BASE/contrib/environment-to-ini
+      - make CI=true LINUX_ARCHS=$${LINUX_ARCHS} release
    environment:
      TAGS: bindata sqlite sqlite_unlock_notify
      DEBIAN_FRONTEND: noninteractive
+      UXPVERSION: 4.0.1
+      LINUX_ARCHS: 'linux/amd64,linux/arm64'
+    secrets:
+      - goproxy_override
+
+  prep-multiarch-binaries:
+    image: golang:1.19 # image reuse to reduce pull times, go not required
+    pull: true
+    environment:
+      fallbackversion: main # the fallback version used by make
+    commands:
+      - mkdir docker/bin
+      - ./docker/map-binaries.sh ./dist/binaries forgejo ${CI_COMMIT_TAG##v}
+      - ./docker/map-binaries.sh ./dist/contrib environment-to-ini ${CI_COMMIT_TAG##v}
+    when:
+      event: tag
+
+  build-docker:
+    image: woodpeckerci/plugin-docker-buildx
+    pull: true
+    settings:
+      dockerfile: Dockerfile.ci
+      platforms: linux/amd64
+      registry:
+        from_secret: domain
+      tag: ${CI_COMMIT_TAG##v}
+      repo: ${CI_REPO_LINK##https://}
+      password:
+        from_secret: releaseteamtoken
+      username:
+        from_secret: releaseteamuser
+    when:
+      event: tag

  gpg-sign:
    image: plugins/gpgsign:1
@ -49,6 +102,8 @@ pipeline:
        - "dist/release/*"
      key:
        from_secret: releaseteamgpg
+    when:
+      event: tag

  release:
    image: golang:1.19
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@ -0,0 +1,52 @@
+FROM codeberg.org/forgejo/alpine:3.17.0
+LABEL maintainer="contact@forgejo.org"
+
+ARG TARGETPLATFORM
+ENV TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64}
+
+EXPOSE 22 3000
+
+#regrouped as updates to it should happen less frequently
+RUN addgroup \
+  -S -g 1000 \
+  git && \
+  adduser \
+  -S -H -D \
+  -h /data/git \
+  -s /bin/bash \
+  -u 1000 \
+  -G git \
+  git && \
+  echo "git:*" | chpasswd -e
+
+RUN apk --no-cache add \
+  bash \
+  ca-certificates \
+  curl \
+  gettext \
+  git \
+  linux-pam \
+  openssh \
+  s6 \
+  sqlite \
+  su-exec \
+  gnupg \
+  tzdata
+
+ENV USER git
+ENV GITEA_CUSTOM /data/gitea
+
+VOLUME ["/data"]
+
+ENTRYPOINT ["/usr/bin/entrypoint"]
+CMD ["/bin/s6-svscan", "/etc/s6"]
+
+COPY docker/root /
+COPY docker/bin/$TARGETPLATFORM/forgejo /app/gitea/gitea
+COPY docker/bin/$TARGETPLATFORM/environment-to-ini /usr/local/bin/environment-to-ini
+#
+# s/755/775/ in the following is to avoid the corrupted layer 4f4fb700ef54
+# https://codeberg.org/Codeberg/Community/issues/800#issue-210309
+#
+RUN chmod 775 /usr/bin/entrypoint /app/gitea/gitea /usr/local/bin/gitea /usr/local/bin/environment-to-ini
+RUN chmod 775 /etc/s6/gitea/* /etc/s6/openssh/* /etc/s6/.s6-svscan/*
--- a/contrib/environment-to-ini/Makefile
+++ b/contrib/environment-to-ini/Makefile
@ -0,0 +1,146 @@
+ifeq ($(USE_REPO_TEST_DIR),1)
+
+# This rule replaces the whole Makefile when we're trying to use /tmp repository temporary files
+location = $(CURDIR)/$(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
+self := $(location)
+
+%:
+	@tmpdir=`mktemp --tmpdir -d` ; \
+	echo Using temporary directory $$tmpdir for test repositories ; \
+	USE_REPO_TEST_DIR= $(MAKE) -f $(self) --no-print-directory REPO_TEST_DIR=$$tmpdir/ $@ ; \
+	STATUS=$$? ; rm -r "$$tmpdir" ; exit $$STATUS
+
+else
+
+# This is the "normal" part of the Makefile
+
+DIST := ../../dist
+DIST_DIRS := $(DIST)/contrib $(DIST)/contrib-release
+
+GO ?= go
+HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
+COMMA := ,
+
+XGO_VERSION := go-1.19.x
+XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest
+
+ifeq ($(HAS_GO), GO)
+	GOPATH ?= $(shell $(GO) env GOPATH)
+	export PATH := $(GOPATH)/bin:$(PATH)
+
+	CGO_CFLAGS ?= $(shell $(GO) env CGO_CFLAGS) $(CGO_EXTRA_CFLAGS)
+endif
+
+EXECUTABLEPREFIX := environment-to-ini
+
+ifeq ($(shell sed --version 2>/dev/null | grep -q GNU && echo gnu),gnu)
+	SED_INPLACE := sed -i
+else
+	SED_INPLACE := sed -i ''
+endif
+
+EXTRA_GOFLAGS ?=
+
+MAKE_VERSION := $(shell "$(MAKE)" -v | head -n 1)
+
+ifeq ($(RACE_ENABLED),true)
+	GOFLAGS += -race
+	GOTESTFLAGS += -race
+endif
+
+STORED_VERSION_FILE := VERSION
+
+ifneq ($(DRONE_TAG),)
+	VERSION ?= $(subst v,,$(DRONE_TAG))
+	GITEA_VERSION ?= $(VERSION)
+else
+	ifneq ($(DRONE_BRANCH),)
+		VERSION ?= $(shell echo $(DRONE_BRANCH) | sed -e 's|v\([0-9.][0-9.]*\)/.*|\1|')
+	else
+		VERSION ?= main
+	endif
+
+	STORED_VERSION=$(shell cat $(STORED_VERSION_FILE) 2>/dev/null)
+	ifneq ($(STORED_VERSION),)
+		GITEA_VERSION ?= $(STORED_VERSION)
+	else
+		GITEA_VERSION ?= $(shell git describe --tags --always | sed 's/-/+/' | sed 's/^v//')
+	endif
+endif
+
+LDFLAGS := $(LDFLAGS) -s -w -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(GITEA_VERSION)" -X "main.Tags=$(TAGS)"
+
+LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64
+
+GO_PACKAGES ?= $(shell $(GO) list ./... | grep -v /vendor/)
+
+TAGS ?=
+TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS))
+
+GO_DIRS := .
+
+GO_SOURCES := $(wildcard *.go)
+GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go")
+
+.PHONY: go-check
+go-check:
+	$(eval MIN_GO_VERSION_STR := $(shell grep -Eo '^go\s+[0-9]+\.[0-9]+' go.mod | cut -d' ' -f2))
+	$(eval MIN_GO_VERSION := $(shell printf "%03d%03d" $(shell echo '$(MIN_GO_VERSION_STR)' | tr '.' ' ')))
+	$(eval GO_VERSION := $(shell printf "%03d%03d" $(shell $(GO) version | grep -Eo '[0-9]+\.[0-9]+' | tr '.' ' ');))
+	@if [ "$(GO_VERSION)" -lt "$(MIN_GO_VERSION)" ]; then \
+		echo "Gitea requires Go $(MIN_GO_VERSION_STR) or greater to build. You can get it at https://go.dev/dl/"; \
+		exit 1; \
+	fi
+
+.PHONY: clean
+clean:
+	$(GO) clean -i ./...
+	rm -rf $(DIST)
+
+.PHONY: generate
+generate: $(TAGS_PREREQ) generate-go
+
+.PHONY: generate-go
+generate-go: $(TAGS_PREREQ)
+	@echo "Running go generate..."
+	@CC= GOOS= GOARCH= $(GO) generate -tags '$(TAGS)' $(GO_PACKAGES)
+
+.PHONY: release
+release: generate release-linux release-compress release-copy
+
+$(DIST_DIRS):
+	mkdir -p $(DIST_DIRS)
+
+.PHONY: release-windows
+release-windows: | $(DIST_DIRS)
+	CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/contrib -tags 'netgo osusergo $(TAGS)' -ldflags '-linkmode external -extldflags "-static" $(LDFLAGS)' -targets 'windows/*' -out $(EXECUTABLEPREFIX)-$(VERSION) .
+ifeq ($(CI),true)
+	cp /build/$(EXECUTABLEPREFIX)-* $(DIST)/contrib
+endif
+
+.PHONY: release-linux
+release-linux: | $(DIST_DIRS)
+	CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/contrib -tags 'netgo osusergo $(TAGS)' -ldflags '-linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(LINUX_ARCHS)' -out $(EXECUTABLEPREFIX)-$(VERSION) .
+ifeq ($(CI),true)
+	cp /build/$(EXECUTABLEPREFIX)-* $(DIST)/contrib
+endif
+
+.PHONY: release-darwin
+release-darwin: | $(DIST_DIRS)
+	CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/contrib -tags 'netgo osusergo $(TAGS)' -ldflags '$(LDFLAGS)' -targets 'darwin-10.12/amd64,darwin-10.12/arm64' -out $(EXECUTABLEPREFIX)-$(VERSION) .
+ifeq ($(CI),true)
+	cp /build/$(EXECUTABLEPREFIX)-* $(DIST)/contrib
+endif
+
+.PHONY: release-compress
+release-compress: | $(DIST_DIRS)
+ifeq ($(CI),true)
+	cd $(DIST); for file in `find ./contrib -type f -name "$(EXECUTABLEPREFIX)-*"`; do upx --best --lzma $${file}; done;
+endif
+
+.PHONY: release-copy
+release-copy: | $(DIST_DIRS)
+	cd $(DIST); for file in `find ./contrib -type f -name "$(EXECUTABLEPREFIX)-*"`; do cp $${file} ./contrib-release/; done;
+
+# This endif closes the if at the top of the file
+endif
--- a/docker/archmap.txt
+++ b/docker/archmap.txt
@ -0,0 +1,9 @@
+# file-name-suffix:docker platform
+# quick lookups: https://github.com/techknowlogick/xgo and https://github.com/BretFisher/multi-platform-docker-build#know-what-platforms-you-can-build-in-your-docker-engine
+linux-amd64:linux/amd64
+linux-386:linux/386
+linux-arm-6:linux/arm/v6
+linux-arm-7:linux/arm/v7
+linux-arm64:linux/arm64
+linux-riscv64:linux/riscv64
+# add mappings for more platforms below
--- a/docker/map-binaries.sh
+++ b/docker/map-binaries.sh
@ -0,0 +1,45 @@
+#!/bin/bash
+
+if [[ ${#@} -lt 2 ]]; then
+  echo "not enough arguments provided" 1>&2
+  exit 1
+fi
+
+BINARYLOOKUPPATH="${1}"
+FILEBASENAME="${2}"
+
+if [[ ${#@} -gt 2 ]] && ! [[ "${3}" == "" ]]; then
+  FILEVERSION="${3}"
+else
+  FILEVERSION="$FALLBACKVERSION"
+fi
+
+FILE_WIHTOUT_PLATFORM="${FILEBASENAME}-${FILEVERSION}"
+
+DOCKERDIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+WS_BASE="${DOCKERDIR%/docker}"
+
+if [[ ! -f "${DOCKERDIR}/archmap.txt" ]]; then
+  echo "no archmap found" 1>&2
+  exit 1
+fi
+
+function map(){
+  local file="${1}"
+  local bin_platform="${file##./$FILE_WIHTOUT_PLATFORM-}"
+  if ! grep -q "$bin_platform" "${DOCKERDIR}/archmap.txt"; then
+    echo "no matching platform for $file found" 2>&1
+    return 0
+  fi
+
+  local platform="$(grep $bin_platform "${DOCKERDIR}/archmap.txt" | cut -d':' -f2)"
+  mkdir -p "${DOCKERDIR}/bin/$platform"
+  echo "Mapped ${file} to $platform, copying"
+  echo cp "${file}" "${DOCKERDIR}/bin/$platform/${FILEBASENAME}"
+}
+
+cd "$WS_BASE/${BINARYLOOKUPPATH}"
+
+while IFS= read -r -d $'\0' infile; do
+  map "$infile"
+done < <(find . -maxdepth 1 -type f -name "${FILE_WIHTOUT_PLATFORM}-*" -print0)