commit a4f32ac1630b7b789623137ab1b75b8149b1879c Author: Colin Finck Date: Sat Sep 7 12:01:16 2019 +0200 Add a configuration for the upcoming Web-Playground BuildBot at https://build.web-content.reactos.org to build and deploy web-content repo changes to a *.web-content.reactos.org subdomain. Untested so far, and only testable when the rest is ready. diff --git a/buildbot/build b/buildbot/build new file mode 100755 index 0000000..d651a30 --- /dev/null +++ b/buildbot/build @@ -0,0 +1,5 @@ +#!/bin/bash +# ReactOS Web-Playground BuildBot Build Scripts +# build - Build the entire website using Hugo (without minification for debugging purposes) + +hugo diff --git a/buildbot/buildbot.tac b/buildbot/buildbot.tac new file mode 100644 index 0000000..10784c0 --- /dev/null +++ b/buildbot/buildbot.tac @@ -0,0 +1,30 @@ +import os + +from twisted.application import service +from buildbot.master import BuildMaster + +basedir = '/srv/buildbot/master_data' +rotateLength = 10000000 +maxRotatedFiles = 10 +configfile = 'master.cfg' + +# Default umask for server +umask = None + +# if this is a relocatable tac file, get the directory containing the TAC +if basedir == '.': + basedir = os.path.abspath(os.path.dirname(__file__)) + +# note: this line is matched against to check that this is a buildmaster +# directory; do not edit it. +application = service.Application('buildmaster') +from twisted.python.logfile import LogFile +from twisted.python.log import ILogObserver, FileLogObserver +logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, + maxRotatedFiles=maxRotatedFiles) +application.setComponent(ILogObserver, FileLogObserver(logfile).emit) + +m = BuildMaster(basedir, configfile, umask) +m.setServiceParent(application) +m.log_rotation.rotateLength = rotateLength +m.log_rotation.maxRotatedFiles = maxRotatedFiles \ No newline at end of file diff --git a/buildbot/credentials.ini b/buildbot/credentials.ini new file mode 100644 index 0000000..e08d895 --- /dev/null +++ b/buildbot/credentials.ini @@ -0,0 +1,10 @@ +[github] +personal_access_token=XXXXX +webhook_secret=XXXXX + +[ldap] +url=XXXXX +basedn=XXXXX +binddn=XXXXX +passwd=XXXXX +search=XXXXX diff --git a/buildbot/deploy b/buildbot/deploy new file mode 100755 index 0000000..6510bf5 --- /dev/null +++ b/buildbot/deploy @@ -0,0 +1,75 @@ +#!/bin/bash +# ReactOS Web-Playground BuildBot Build Scripts +# deploy - Make the built website available at a subdomain of *.web-content.reactos.org if the commit author is trusted +# +# Parameter $1 - Branch that has been built. +# Parameter $2 - Author of the changes. This is guaranteed to be a GitHub username if (and only if!) we built a PR. + +####################################### +# FUNCTIONS +####################################### + +# Returns 0 (true) if the given GitHub username belongs to the given GitHub Team ID, otherwise returns 1 (false). +check_team_membership() { + local member=$1 + local team_id=$2 + + local status=`curl -H "Authorization: token ${personal_access_token}" -i -s https://api.github.com/teams/${team_id}/memberships/${member} | head -n 1 | cut -d$' ' -f2` + if [[ "${status}" == "200" ]]; then + return 0 + else + return 1 + fi +} + + +####################################### +# ENTRY POINT +####################################### + +if [[ "$2" == "" ]]; then + echo "Please specify the branch and author!" + exit 1 +fi + +branch=$1 +author=$2 + +# Read secret configuration from credentials.ini. +# Doesn't take INI sections into account, but sufficient for our use case. +source <(grep personal_access_token credentials.ini) + +# These IDs are no secret and can be retrieved by an API call to https://api.github.com/orgs/reactos/teams +developer_group_id=2505522 # "ReactOS Developer Team" +trusted_group_id=3404185 # "Trusted Web-Playground Users" + + +# What branch are we building? +if [[ "${branch}" == "master" ]]; then + # A push to master is only possible by the developer team, so we don't have to check any team memberships. + subdomain="master" +elif [[ "${branch}" =~ ^refs/pull/([0-9]+)/ ]]; then + # Everyone can open a Pull Request against our repository, but we cannot trust everyone to not abuse this system and use it to set up a malicious/spam website. + subdomain="pr${BASH_REMATCH[1]}" + + # Check if the author is either part of the developer team or the additional group of trusted people. + if ! { check_team_membership ${author} ${developer_group_id} || check_team_membership ${author} ${trusted_group_id} }; then + echo "This script would usually deploy the build result of your Pull Request to https://${subdomain}.web-content.reactos.org" + echo "However, to prevent abuse, your GitHub account first needs to be added to the trusted user list." + echo "Ask someone from the developer team if you need this deployment feature and want to be added to the trusted user list." + exit 0 + fi +else + # Anything else (like a local branch) is not deployed. + echo "Only changes to master and Pull Requests are deployed." + exit 0 +fi + + +# The author is trusted, so move over the built files to deploy the website. +target="/srv/www/web-content/${subdomain}" +rm -rf ${target} +mv public ${target} || exit 1 + +echo "Congratulations! The build result has been deployed to https://${subdomain}.web-content.reactos.org" +exit 0 diff --git a/buildbot/master.cfg b/buildbot/master.cfg new file mode 100644 index 0000000..fd85463 --- /dev/null +++ b/buildbot/master.cfg @@ -0,0 +1,170 @@ +# -*- python -*- +# ex: set filetype=python: + +from buildbot.plugins import worker, schedulers, util, steps, reporters +from buildbot.www import auth +from twisted.internet import defer + +import configparser +import pathlib + + +class ReactOSLDAPUserInfoProvider(auth.UserInfoProviderBase): + def getUserInfo(self, username): + # ReactOSLDAPAuth already checks for group membership, so every logged in user can be added to the 'logged_in_developer' group. + return defer.succeed({ + 'userName': username, + 'email': username, + 'groups': ['logged_in_developer'], + }) + +class ReactOSLDAPAuth(auth.CustomAuth): + def __init__(self): + super().__init__() + self.userInfoProvider = ReactOSLDAPUserInfoProvider() + + global credentials + self.url = credentials.get("ldap", "url") + self.basedn = credentials.get("ldap", "basedn") + self.binddn = credentials.get("ldap", "binddn") + self.passwd = credentials.get("ldap", "passwd") + self.search = credentials.get("ldap", "search") + + def check_credentials(self, user, password): + import ldap + + user = user.decode('utf-8') + password = password.decode('utf-8') + + try: + res = self._authenticate(user, password) + except ldap.LDAPError as e: + print('ReactOSLDAPAuth: LDAPError: ' + str(e)) + res = False + + return res + + def _authenticate(self, user, password): + import ldap + + search_conn = ldap.initialize(self.url) + search_conn.simple_bind_s(self.binddn, self.passwd) + + try: + result = search_conn.search_s(self.basedn, ldap.SCOPE_SUBTREE, self.search % user, ['objectclass'], 1) + except ldap.SERVER_DOWN: + print('ReactOSLDAPAuth: LDAP server seems down') + return False + + search_conn.unbind() + + # Make sure we found a single user in the LDAP DB + if not result or len(result) != 1: + print('ReactOSLDAPAuth: User "%s" not found in LDAP DB' % user) + return False + + auth_conn = ldap.initialize(self.url) + ldap_dn = result[0][0] + + # Authenticate the user + try: + auth_conn.simple_bind_s(ldap_dn, password) + except ldap.INVALID_CREDENTIALS: + print('ReactOSLDAPAuth: Invalid credentials for user "%s"' % user) + return False + + auth_conn.unbind() + return True + + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} +c['buildbotNetUsageData'] = None + +# Read the config +credentials = configparser.RawConfigParser() +if len(credentials.read("credentials.ini")) == 0: + raise Exception("credentials.ini not found or empty!") + +####### WORKERS + +c['workers'] = [ + worker.LocalWorker('web-playground'), +] + +####### SCHEDULERS + +c['schedulers'] = [] + +c['schedulers'].append(schedulers.AnyBranchScheduler( + name='all', + builderNames=["Build"], +)) + +c['schedulers'].append(schedulers.ForceScheduler( + name="Force", + reason=util.StringParameter(name="reason", required=True, label="Reason:", default="A reason for running this build"), + codebases=[util.CodebaseParameter("", repository=None, project=None)], + builderNames=["Build"], +)) + +####### BUILDERS + +scripts_root = "../../" + +Build = util.BuildFactory() +Build.addStep(steps.GitHub(repourl='https://github.com/reactos/web-content.git', mode='full', haltOnFailure=True)) +Build.addStep(steps.ShellCommand(name="build", command=[scripts_root + "build"], haltOnFailure=True)) +Build.addStep(steps.ShellCommand(name="deploy", command=[scripts_root + "deploy", util.WithProperties('%(branch)s'), util.WithProperties('%(author)s')], haltOnFailure=True)) + +c['builders'] = [ + util.BuilderConfig(name="Build", workernames=["web-playground"], builddir="Build", factory=Build), +] + +####### BUILDBOT SERVICES + +c['services'] = [ + reporters.GitHubStatusPush( + token=credentials.get("github", "personal_access_token") + ), +] + +####### PROJECT IDENTITY + +c['projectName'] = "ReactOS Web-Playground" +c['projectURL'] = "https://web-content.reactos.org/" +c['titleURL'] = "https://build.web-content.reactos.org/" +c['buildbotURL'] = "https://build.web-content.reactos.org/" + +# web UI +c['www'] = { + 'port': 8010, + 'plugins': { + 'waterfall_view': True, + 'console_view': True, + 'grid_view': True, + }, + 'auth': ReactOSLDAPAuth(), + 'authz': util.Authz( + allowRules=[ + util.AnyControlEndpointMatcher(role="logged_in_developer"), + ], + roleMatchers=[ + util.RolesFromGroups(), + ] + ), + 'change_hook_dialects': { + 'github': { + 'secret': credentials.get("github", "webhook_secret"), + }, + }, +} + +####### DB URL + +c['db'] = { + # This specifies what database buildbot uses to store its state. You can leave + # this at its default for all but the largest installations. + 'db_url': "sqlite:///state.sqlite", +} diff --git a/buildbot/requirements.txt b/buildbot/requirements.txt new file mode 100644 index 0000000..1ea07c2 --- /dev/null +++ b/buildbot/requirements.txt @@ -0,0 +1,3 @@ +buildbot[bundle,tls]==2.4.0 +python-ldap==3.2.0 +treq==18.6.0 diff --git a/www/index.htm b/www/index.htm new file mode 100644 index 0000000..4a679ec --- /dev/null +++ b/www/index.htm @@ -0,0 +1,24 @@ + + + + + ReactOS Web-Playground + + + + +

Welcome to the ReactOS Web-Playground

+ +

This machine provides resources to help people improving the ReactOS Website.

+ +

+ You can start by cloning the web-content repository.
+ Whenever you submit a Pull Request, it is built by our Web-Playground BuildBot and the result is made available at a *.web-content.reactos.org subdomain. +

+ +

Check out master.web-content.reactos.org for the latest master branch of the website.

+ + +