finalize automation/CI workflows:

- build markdown/HTML versions, use separate makefile targets/hecat configuration files for each export format/push task
- add non-free.md generation step to markdown export hecat configuration file
- add makefile targets to push exports to https://github.com/awesome-selfhosted/awesome-selfhosted and https://github.com/nodiscc/awesome-selfhosted-html-preview
- install: install sphinx <7 directly using pip since it can't be installed from setup.py https://github.com/sphinx-doc/sphinx/issues/11130
- pin sphinx to version <7, sphinx-design is not compatible with sphinx 7 https://github.com/executablebooks/sphinx-design/issues/130
- add sphinx configuration file
- tools/makefile: monkeypatch furo theme to display build tools and license link in footer (furo/sphinx does not provide a way to customize this https://github.com/pradyunsg/furo/discussions/612)
- fix dependencies between makefile targets
- setup/document SSH deploy keys usage
- each repository requires a different deploy key, trying to add a deploy key to repo B, which has already been added to repo A results in error 'this key already exists'
- set git remote URL to use ssh URIs before push (required to use SSH deploy keys)
- when referencing an environment secret, environment: key must be defined explicitly for the step (https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#referencing-an-environment)
- use a single SSH key file/no ssh-agent, else git push will try to use the first registered key and fail with permission denied when trying to push to the HTML repository
- use variables to identify target repos
- use temporary repositories as push destination

.github/workflows/build.yml

@@ -9,10 +9,35 @@ concurrency:
   group: build-${{ github.ref }}
   cancel-in-progress: true
 
+# This job requires deploy keys to be added to target repositories (https://docs.github.com/en/authentication/connecting-to-github-with-ssh/managing-deploy-keys)
+# Generate deploy keys locally:
+# $ ssh-keygen -t rsa -f awesome-selfhosted-deploy-key -C awesome-selfhosted-deploy-key
+# $ ssh-keygen -t rsa -f awesome-selfhosted-html-deploy-key -C awesome-selfhosted-html-deploy-key
+# Paste the contents of awesome-selfhosted-deploy-key.pub to https://github.com/nodiscc/awesome-selfhosted/settings/keys/new, name: awesome-selfhosted-deploy-key, allow write access
+# Paste the contents of awesome-selfhosted-html-deploy-key.pub to https://github.com/nodiscc/awesome-selfhosted-html-preview/settings/keys/new, name: awesome-selfhosted-html-deploy-key
+# Access https://github.com/awesome-selfhosted/awesome-selfhosted-data/settings/environments, create new environment with name: production
+#   - deployment branches: selected branches
+#   - add deployment branch rule: branch name pattern: master
+#   - environment secrets: add secret with name: SSH_MARKDOWN_DEPLOY_KEY, and value: paste the contents of awesome-selfhosted-deploy-key
+#   - environment secrets: add secret with name: SSH_HTML_DEPLOY_KEY, and value: paste the contents of awesome-selfhosted-html-deploy-key
+
 jobs:
   build:
+    environment: production
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - run: make awesome_lint
-      - run: make export
+      - run: make export_markdown export_html
+      - name: setup markdown repository SSH deploy key
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.SSH_MARKDOWN_DEPLOY_KEY }}" > ~/.ssh/id_rsa
+          chmod 0600 ~/.ssh/id_rsa
+      - run: make push_markdown
+      - name: setup HTML repository SSH deploy key
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.SSH_HTML_DEPLOY_KEY }}" > ~/.ssh/id_rsa
+          chmod 0600 ~/.ssh/id_rsa
+      - run: make push_html

.github/workflows/daily-checks.yml

@@ -1,4 +1,4 @@
-name: dead links and unmaintained projects checks
+name: dead links/unmaintained projects checks
 
 on:
   schedule:

.github/workflows/pull-request.yml

@@ -14,4 +14,4 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - run: make awesome_lint
-      - run: make export
+      - run: make export_markdown

.gitignore

@@ -1,2 +1,4 @@
 awesome-selfhosted/
+awesome-selfhosted-html-preview/
+html/
 .venv/

.hecat/export-html.yml

@@ -1,12 +1,9 @@
 steps:
-  - name: export YAML data to awesome-selfhosted markdown
+  - name: export YAML data to multi-page markdown site
-    module: exporters/markdown_singlepage
+    module: exporters/markdown_multipage
     module_options:
       source_directory: ./
-      output_directory: awesome-selfhosted
+      output_directory: html
-      output_file: README.md
-      back_to_top_url: '#awesome-selfhosted'
-      # authors_file: AUTHORS.md
       exclude_licenses:
         - '⊘ Proprietary'
         - 'BUSL-1.1'
@@ -16,3 +13,6 @@ steps:
         - 'Commons-Clause'
         - 'DPL'
         - 'SSPL-1.0'
+        - 'DPL'
+        - 'Elastic-1.0'
+        - 'Elastic-2.0'

.hecat/export-markdown.yml

@@ -0,0 +1,46 @@
+steps:
+  - name: export YAML data to single-page markdown
+    module: exporters/markdown_singlepage
+    module_options:
+      source_directory: ./
+      output_directory: awesome-selfhosted
+      output_file: README.md
+      markdown_header: markdown/header.md
+      markdown_footer: markdown/footer.md
+      back_to_top_url: '#awesome-selfhosted'
+      exclude_licenses:
+        - '⊘ Proprietary'
+        - 'BUSL-1.1'
+        - 'CC-BY-NC-4.0'
+        - 'CC-BY-NC-SA-3.0'
+        - 'CC-BY-ND-3.0'
+        - 'Commons-Clause'
+        - 'DPL'
+        - 'SSPL-1.0'
+        - 'DPL'
+        - 'Elastic-1.0'
+        - 'Elastic-2.0'
+
+  - name: export YAML data to single-page markdown (non-free)
+    module: exporters/markdown_singlepage
+    module_options:
+      source_directory: ./
+      output_directory: awesome-selfhosted
+      output_file: non-free.md
+      markdown_header: markdown/non-free-header.md
+      licenses_file: licenses-nonfree.yml
+      back_to_top_url: '##awesome-selfhosted---non-free-software'
+      render_empty_categories: False
+      render_category_headers: False
+      include_licenses:
+        - '⊘ Proprietary'
+        - 'BUSL-1.1'
+        - 'CC-BY-NC-4.0'
+        - 'CC-BY-NC-SA-3.0'
+        - 'CC-BY-ND-3.0'
+        - 'Commons-Clause'
+        - 'DPL'
+        - 'SSPL-1.0'
+        - 'DPL'
+        - 'Elastic-1.0'
+        - 'Elastic-2.0'

Makefile

@@ -1,15 +1,18 @@
+# requirements: git bash make python3-pip python3-venv
 SHELL := /bin/bash
+MARKDOWN_REPOSITORY = nodiscc/awesome-selfhosted
+HTML_REPOSITORY = nodiscc/awesome-selfhosted-html-preview
 
 .PHONY: install # install build tools in a virtualenv
 install:
 	python3 -m venv .venv
 	source .venv/bin/activate && \
 	pip3 install wheel && \
-	pip3 install --force git+https://github.com/nodiscc/hecat.git@master
+	pip3 install --force git+https://github.com/nodiscc/hecat.git@master 'sphinx<7'
 
 .PHONY: import # import data from original list at https://github.com/awesome-selfhosted/awesome-selfhosted
-import: install
+import: clean install
-	rm -rf awesome-selfhosted && git clone --depth=1 https://github.com/awesome-selfhosted/awesome-selfhosted
+	git clone --depth=1 https://github.com/awesome-selfhosted/awesome-selfhosted
 	cp awesome-selfhosted/.github/.mailmap .mailmap
 	cp awesome-selfhosted/AUTHORS AUTHORS
 	rm -rf tags/ software/ platforms/
@@ -27,13 +30,37 @@ awesome_lint: install
 	source .venv/bin/activate && \
 	hecat --config .hecat/awesome-lint.yml
 
-.PHONY: export # export markdown singlepage document from yaml data
+.PHONY: export_markdown # render markdown export from YAML data (https://github.com/awesome-selfhosted/awesome-selfhosted)
-export: install
+export_markdown: install
-	rm -rf awesome-selfhosted && git clone https://github.com/awesome-selfhosted/awesome-selfhosted
+	rm -rf awesome-selfhosted/
-	source .venv/bin/activate && \
+	git clone https://github.com/$(MARKDOWN_REPOSITORY)
-	hecat --config .hecat/export.yml
+	source .venv/bin/activate && hecat --config .hecat/export-markdown.yml
 	cd awesome-selfhosted && git diff --color=always
 
+.PHONY: export_html # render HTML export from YAML data (https://nodiscc.github.io/awesome-selfhosted-html-preview/)
+export_html: install
+	rm -rf awesome-selfhosted-html-preview/ html/
+	git clone https://github.com/$(HTML_REPOSITORY)
+	mkdir html && source .venv/bin/activate && hecat --config .hecat/export-html.yml
+	sed -i 's|<a href="https://github.com/pradyunsg/furo">Furo</a>|<a href="https://github.com/nodiscc/hecat/">hecat</a>, <a href="https://www.sphinx-doc.org/">sphinx</a> and <a href="https://github.com/pradyunsg/furo">furo</a>. Content under <a href="https://github.com/awesome-selfhosted/awesome-selfhosted-data/blob/master/LICENSE">CC-BY-SA 3.0</a> license.|' .venv/lib/python*/site-packages/furo/theme/furo/page.html
+	source .venv/bin/activate && sphinx-build -b html -c ./ html/md/ html/html/
+	rm -rf html/html/.buildinfo html/html/objects.inv html/html/.doctrees awesome-selfhosted-html-preview/*
+
+.PHONY: push_markdown # commit and push changes to the markdown repository
+push_markdown:
+	cd awesome-selfhosted && git remote set-url origin git@github.com:$(MARKDOWN_REPOSITORY)
+	cd awesome-selfhosted && git config user.name awesome-selfhosted-bot && git config user.email github-actions@github.com
+	cd awesome-selfhosted && git add . && (git diff-index --quiet HEAD || git commit -m "[bot] build markdown repository")
+	cd awesome-selfhosted && git push -f
+
+.PHONY: push_html # commit and push changes to the HTML site repository (amend previous commit and force-push)
+push_html:
+	mv html/html/* awesome-selfhosted-html-preview/
+	cd awesome-selfhosted-html-preview/ && git remote set-url origin git@github.com:$(HTML_REPOSITORY)
+	cd awesome-selfhosted-html-preview/ && git config user.name awesome-selfhosted-bot && git config user.email github-actions@github.com
+	cd awesome-selfhosted-html-preview/ && git add . && (git diff-index --quiet HEAD || git commit --amend -m "[bot] build HTML site")
+	cd awesome-selfhosted-html-preview/ && git push -f
+
 .PHONY: url_check # check URLs for dead links or other connection problems
 url_check: install
 	source .venv/bin/activate && \
@@ -46,7 +73,7 @@ authors:
 
 .PHONY: clean # clean files generated by automated tasks
 clean:
-	rm -rf awesome-selfhosted
+	rm -rf awesome-selfhosted/ awesome-selfhosted-html-preview/ html/
 
 .PHONY: help # generate list of targets with descriptions
 help:

README.md

@@ -83,7 +83,10 @@ install             install build tools in a virtualenv
 import              import data from original list at https://github.com/awesome-selfhosted/awesome-selfhosted
 update_metadata     update metadata from project repositories/API
 awesome_lint        check data against awesome-selfhosted guidelines
-export              export markdown singlepage document from yaml data
+export_markdown     render markdown export from YAML data (https://github.com/awesome-selfhosted/awesome-selfhosted)
+export_html         render HTML export from YAML data (https://nodiscc.github.io/awesome-selfhosted-html-preview/)
+push_markdown       commit and push changes to the markdown repository
+push_html           commit and push changes to the HTML site repository (amend previous commit and force-push)
 url_check           check URLs for dead links or other connection problems
 authors             update the AUTHORS file
 clean               clean files generated by automated tasks

conf.py

@@ -0,0 +1,49 @@
+# Configuration file for the Sphinx documentation builder.
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+project = 'awesome-selfhosted'
+author = 'awesome-selfhosted community'
+version = '1.0.0'
+release = '1.0.0'
+copyright = '2015-2023, the awesome-selfhosted community'
+language = 'en'
+html_title = 'awesome-selfhosted'
+html_theme = 'furo'
+html_show_sphinx = False
+html_show_search_summary = True
+html_copy_source = False
+html_show_copyright = True
+html_use_opensearch = 'https://nodiscc.github.io/awesome-selfhosted-html-preview/'
+html_favicon = '_static/favicon.ico'
+html_logo = '_static/logo.svg'
+extensions = ['myst_parser', 'sphinx_design']
+source_suffix = ['.md']
+templates_path = ['_templates']
+exclude_patterns = []
+html_static_path = ['_static']
+
+# myst-parser configuration (https://myst-parser.readthedocs.io/en/latest/configuration.html)
+myst_enable_extensions = ['fieldlist']
+myst_html_meta = {
+    "description lang=en": "A list of Free Software network services and web applications which can be hosted on your own servers",
+    "charset": "UTF-8"
+}
+
+# theme configuration (https://pradyunsg.me/furo/customisation/)
+html_theme_options = {
+    "top_of_page_button": None,
+    # "announcement": "Example announcement!"
+    "source_repository": "https://github.com/awesome-selfhosted/awesome-selfhosted-data",
+    "source_branch": "master",
+    "footer_icons": [
+        {
+            "name": "GitHub",
+            "url": "https://github.com/awesome-selfhosted/awesome-selfhosted-data",
+            "html": """
+                <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 16 16">
+                    <path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path>
+                </svg>
+            """,
+            "class": "",
+        },
+    ]
+}