Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • cs/util/inau
  • lorenzo.pivetta/inau
2 results
Show changes
Commits on Source (52)
*
Aggiungere Neable/Disable flag per i repository ed implementare lo logica conseguente
Rivedere /files sembra proprio sbagliata
Testare server Python (con annessa la gestione di più file)
Integrazione con system per l'avvio ed il riavvio automatico
Rendere visibili (in sola lettura) le builds (e magari anche i relativi artifacts)
Implementare mode=limit se necessario
File added
File added
{"object_kind":"tag_push","event_name":"tag_push","before":"0000000000000000000000000000000000000000","after":"d56ffed6a0e25a046f4debd0451ebd606286a578","ref":"refs/tags/1.0.0[USED]","checkout_sha":"ff00ed95be0e2c6737e3d96b19051b2dc339c0db","message":"v1.0.0","user_id":13,"user_name":"Alessio Igor Bogani","user_username":"alessio.bogani"[USED],"user_email":""[USED],"user_avatar":"https://secure.gravatar.com/avatar/f13312e20198c46ed5356c9c7fa1bc2a?s=80\u0026d=identicon","project_id":297,"project":{"id":297,"name":"fake","description":"Fake device, just for testing","web_url":"https://gitlab.elettra.eu/cs/ds/fake","avatar_url":null,"git_ssh_url":"git@gitlab.elettra.eu:cs/ds/fake.git","git_http_url":"https://gitlab.elettra.eu/cs/ds/fake.git","namespace":"ds","visibility_level":20,"path_with_namespace":"cs/ds/fake[USED]","default_branch":"master","ci_config_path":null,"homepage":"https://gitlab.elettra.eu/cs/ds/fake","url":"git@gitlab.elettra.eu:cs/ds/fake.git","ssh_url":"git@gitlab.elettra.eu:cs/ds/fake.git[USED]","http_url":"https://gitlab.elettra.eu/cs/ds/fake.git"},"commits":[{"id":"ff00ed95be0e2c6737e3d96b19051b2dc339c0db","message":"CHange 2\n","title":"CHange 2","timestamp":"2021-03-17T14:30:24+01:00","url":"https://gitlab.elettra.eu/cs/ds/fake/-/commit/ff00ed95be0e2c6737e3d96b19051b2dc339c0db","author":{"name":"Alessio Igor Bogani","email":"alessio.bogani@elettra.eu[USED]"},"added":[],"modified":["src/Fake.cpp"],"removed":[]}],"total_commits_count":1,"push_options":{},"repository":{"name":"fake","url":"git@gitlab.elettra.eu:cs/ds/fake.git","description":"Fake device, just for testing","homepage":"https://gitlab.elettra.eu/cs/ds/fake","git_http_url":"https://gitlab.elettra.eu/cs/ds/fake.git","git_ssh_url":"git@gitlab.elettra.eu:cs/ds/fake.git","visibility_level":20}}
header "Accept: text/plain"
write-out \n
silent
#!/usr/bin/env python3
from http.server import BaseHTTPRequestHandler, HTTPServer
from http import HTTPStatus
import ssl
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, exc
from multiprocessing import Process, Queue
from enum import Enum, IntEnum
import os
import signal
import json
import sys
import logging
import logging.handlers
import argparse
import datetime
import subprocess
import paramiko
import hashlib
import shutil
from smtplib import SMTP
from email.mime.text import MIMEText
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, DateTime, Boolean, Text, ForeignKey, func
from sqlalchemy.orm import relationship
from enum import Enum, IntEnum
import datetime
Session = sessionmaker()
allbuilders = {}
users = {}
Base = declarative_base()
class Users(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
name = Column(String(255), unique=True, nullable=False)
admin = Column(Boolean, default=False, nullable=False)
notify = Column(Boolean, default=False, nullable=False)
class Architectures(Base):
__tablename__ = 'architectures'
id = Column(Integer, primary_key=True)
name = Column(String(255), unique=True, nullable=False)
platforms = relationship('Platforms', back_populates='architecture')
class Distributions(Base):
__tablename__ = 'distributions'
id = Column(Integer, primary_key=True)
name = Column(String(255), nullable=False)
version = Column(String(255), nullable=False)
platforms = relationship('Platforms', back_populates='distribution')
class Platforms(Base):
__tablename__ = 'platforms'
id = Column(Integer, primary_key=True)
distribution_id = Column(Integer, ForeignKey('distributions.id'), nullable=False)
architecture_id = Column(Integer, ForeignKey('architectures.id'), nullable=False)
architecture = relationship('Architectures', back_populates='platforms')
distribution = relationship('Distributions', back_populates='platforms')
class Builders(Base):
__tablename__ = 'builders'
id = Column(Integer, primary_key=True)
platform_id = Column(Integer, ForeignKey('platforms.id'), nullable=False)
name = Column(String(255), unique=False, nullable=False)
environment = Column(String(255), unique=False, nullable=True)
class Providers(Base):
__tablename__ = 'providers'
id = Column(Integer, primary_key=True)
url = Column(String(255), unique=True, nullable=False)
class RepositoryType(IntEnum):
cplusplus = 0,
python = 1,
configuration = 2,
shellscript = 3,
library = 4
class Repositories(Base):
__tablename__ = 'repositories'
id = Column(Integer, primary_key=True)
provider_id = Column(Integer, ForeignKey('providers.id'), nullable=False)
platform_id = Column(Integer, ForeignKey('platforms.id'), nullable=False)
type = Column(Integer, nullable=False)
name = Column(String(255), nullable=False)
destination = Column(String(255), nullable=False)
enabled = Column(Boolean, default=True, nullable=False)
class Builds(Base):
__tablename__ = 'builds'
id = Column(Integer, primary_key=True)
repository_id = Column(Integer, ForeignKey('repositories.id'), nullable=False)
platform_id = Column(Integer, ForeignKey('platforms.id'), nullable=False)
tag = Column(String(255), nullable=False)
date = Column(DateTime, default=datetime.datetime.now, nullable=False)
status = Column(Integer, nullable=True)
output = Column(Text, nullable=True)
class Artifacts(Base):
__tablename__ = 'artifacts'
id = Column(Integer, primary_key=True)
build_id = Column(Integer, ForeignKey('builds.id'), nullable=False)
hash = Column(String(255), nullable=True)
filename = Column(String(255), nullable=False)
symlink_target = Column(String(255), nullable=True)
def __sendEmail(to_addrs, subject, body):
if to_addrs:
with SMTP(args.smtpserver + "." + args.smtpdomain, port=25) as smtpClient:
sender = args.smtpsender + "@" + args.smtpdomain
msg = MIMEText(body)
msg['Subject'] = "INAU. " + subject
msg['From'] = sender
msg['To'] = ', '.join(to_addrs)
d = smtpClient.sendmail(from_addr=sender, to_addrs=list(to_addrs), msg=msg.as_string())
print("Email sent to ", list(to_addrs), d)
def sendEmail(recipients, subject, body):
notifiable = set()
for user in users:
if (user.notify==True):
notifiable.add(user.name + "@" + args.smtpdomain)
to_addrs = set(recipients).intersection(notifiable)
__sendEmail(to_addrs, subject, body)
def sendEmailAdmins(subject, body):
to_addrs = set()
for user in users:
if (user.admin==True):
to_addrs.add(user.name + "@" + args.smtpdomain)
__sendEmail(to_addrs, subject, body)
class Die:
pass
class Update:
def __init__(self, repository_name, repository_url, build_tag, default_branch):
self.repository_name = repository_name
self.repository_url = repository_url
self.build_tag = build_tag
self.default_branch = default_branch
class Build(Update):
def __init__(self, repository_name, repository_url, build_tag, default_branch):
Update.__init__(self, repository_name, repository_url, build_tag, default_branch)
self.status = ''
self.output = ''
class Store(Build):
def __init__(self, repository_name, repository_url, build_tag, repository_id, repository_type, emails, default_branch):
Build.__init__(self, repository_name, repository_url, build_tag, default_branch)
self.repository_id = repository_id
self.repository_type = repository_type
self.emails = emails
class Builder:
def __init__(self, name, platform_id, environment):
self.name = name
self.platform_id = platform_id
self.platdir = args.repo + '/' + str(platform_id)
if environment is None:
self.environment = ""
else:
self.environment = "source " + environment + "; "
self.queue = Queue()
self.process = Process(target=self.handler)
self.process.start()
def update(self, job):
logger.info("[" + self.name + "] Checkouting " + job.build_tag + " from " + job.repository_url + "...")
builddir = self.platdir + "/" + job.repository_name
buildcmd = ""
if not os.path.isdir(self.platdir):
os.mkdir(self.platdir)
if not os.path.isdir(self.platdir + "/cs/ds/makefiles"):
buildcmd += "git clone --recurse-submodule https://gitlab.elettra.eu/cs/ds/makefiles.git " + self.platdir + "/cs/ds/makefiles"
else:
buildcmd += "git -C " + self.platdir + "/cs/ds/makefiles remote update"
buildcmd += " && git -C " + self.platdir + "/cs/ds/makefiles reset --hard origin/master"
if not os.path.isdir(builddir):
buildcmd += " && git clone --recurse-submodule " + job.repository_url + " " + builddir
buildcmd += " && git -C " + builddir + " remote update"
buildcmd += " && git -C " + builddir + " pull --tags"
buildcmd += " && git -C " + builddir + " branch --no-color --contains " + job.build_tag + " | grep '\<" + job.default_branch + "\>'"
buildcmd += " && git -C " + builddir + " reset --hard " + job.build_tag + " --"
buildcmd += " && git -C " + builddir + " submodule update --init --force --recursive"
subprocess.run(buildcmd, shell=True, check=True)
def build(self, job):
logging.info("[" + self.name + "] Building " + job.build_tag + " from " + job.repository_url + "...")
builddir = self.platdir + "/" + job.repository_name
with paramiko.SSHClient() as sshClient:
sshClient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
sshClient.connect(hostname=self.name, port=22, username="inau",
key_filename="/home/inau/.ssh/id_rsa.pub")
if job.repository_type != RepositoryType.library:
_, raw, _ = sshClient.exec_command("(" + self.environment + "source /etc/profile; cd " + builddir
+ "; make -j`getconf _NPROCESSORS_ONLN`) 2>&1")
else:
_, raw, _ = sshClient.exec_command("(" + self.environment + "source /etc/profile; cd " + builddir
+ "; make -j`getconf _NPROCESSORS_ONLN` && rm -fr .install && PREFIX=.install make install) 2>&1")
job.status = raw.channel.recv_exit_status()
job.output = raw.read().decode('latin-1') # FIXME utf-8 is rejected by Mysql despite it is properly configured
def store(self, job):
logging.info("[" + self.name + "] Storing " + job.build_tag + " from " + job.repository_url + "...")
build = Builds(repository_id=job.repository_id, platform_id=self.platform_id, tag=os.path.basename(job.build_tag),
status=job.status, output=job.output)
self.session.add(build)
self.session.commit()
builddir = self.platdir + "/" + job.repository_name
outcome = job.repository_name + " " + os.path.basename(job.build_tag)
if job.status != 0:
outcome += ": built failed on " + self.name
else:
outcome += ": built successfully on " + self.name
if job.repository_type == RepositoryType.cplusplus or job.repository_type == RepositoryType.python \
or job.repository_type == RepositoryType.shellscript:
basedir = builddir + "/bin/"
elif job.repository_type == RepositoryType.configuration:
basedir = builddir + "/etc/"
elif job.repository_type == RepositoryType.library:
basedir = builddir + "/.install/"
else:
raiseException('Invalid type')
artifacts = []
for rdir_abs, _, fnames in os.walk(basedir):
for fname in fnames:
fname_abs = os.path.join(rdir_abs, fname)
rdir_rel = os.path.relpath(rdir_abs, basedir)
fname_rel = os.path.join(rdir_rel, fname)
if not os.path.islink(fname_abs):
with open(fname_abs,"rb") as fd:
bytes = fd.read()
hashFile = hashlib.sha256(bytes).hexdigest()
hashDir = hashFile[0:2] + "/" + hashFile[2:4]
if not os.path.isdir(args.store + hashDir):
os.makedirs(args.store + hashDir, exist_ok=True)
if not os.path.isfile(args.store + hashFile):
shutil.copyfile(fname_abs, args.store + hashDir + "/" + hashFile, follow_symlinks=False)
artifacts.append(Artifacts(build_id=build.id, hash=hashFile, filename=fname_rel))
else:
artifacts.append(Artifacts(build_id=build.id, filename=fname_rel,
symlink_target=os.path.join(rdir_rel, os.readlink(fname_abs))))
self.session.add_all(artifacts)
self.session.commit()
sendEmail(job.emails, outcome, job.output)
def handler(self):
logger.info("[" + self.name + "] Starting process for builder " + self.name + "...")
engine.dispose()
self.session = Session()
while True:
try:
job = self.queue.get()
if isinstance(job, Die):
logger.info("[" + self.name + "] Stopping process for builder " + self.name + "...")
break
if isinstance(job, Update):
self.update(job)
if isinstance(job, Build):
self.build(job)
if isinstance(job, Store):
self.store(job)
except subprocess.CalledProcessError as c:
sendEmailAdmins("Subprocess failed", str(c))
logger.error("Subprocess failed: ", str(c))
self.session.rollback()
except Exception as e:
sendEmailAdmins("Generic error", str(e))
logger.error("Generic error: ", str(e))
self.session.rollback()
except KeyboardInterrupt as k:
self.session.rollback()
break
finally:
self.session.close()
def signalHandler(signalNumber, frame):
reconcile()
def reconcile():
logger.info('Reconciling...')
session = Session()
try:
global allbuilders
global users
users = session.query(Users).all()
newbuilders = {}
oldbuilders = allbuilders
for b in session.query(Builders).all():
try:
newbuilders[b.platform_id].append(Builder(b.name, b.platform_id, b.environment))
except KeyError:
newbuilders[b.platform_id] = [Builder(b.name, b.platform_id, b.environment)]
allbuilders = newbuilders
for oldbuilder in oldbuilders.values():
for b in oldbuilder:
b.queue.put(Die())
b.process.join()
except Exception as e:
sendEmailAdmins("Reconcilation failed", str(e))
logger.error("Reconciliation failed: ", str(e))
session.rollback()
finally:
session.close()
class Server(BaseHTTPRequestHandler):
def do_POST(self):
engine.dispose()
session = Session()
try:
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length)
if self.headers['Content-Type'] != 'application/json':
self.send_response(HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value)
self.end_headers()
return
post_json = json.loads(post_data.decode('utf-8'))
logger.debug(post_json)
print(post_json)
# Tag deletion
if post_json['after'] == '0000000000000000000000000000000000000000':
self.send_response(HTTPStatus.OK.value)
self.end_headers()
return
# Check if the tag is lightweight
if post_json['after'] == post_json['commits'][0]['id']:
self.send_response(HTTPStatus.OK.value)
self.end_headers()
return
for r in session.query(Repositories).filter(Repositories.name==post_json['project']['path_with_namespace']).all():
if self.headers['X-Gitlab-Event'] == 'Tag Push Hook' and post_json['event_name'] == 'tag_push' and r.enabled:
job = Store(repository_name = r.name, repository_url = post_json['project']['ssh_url'], build_tag=post_json['ref'],
repository_id = r.id, repository_type = r.type, emails=[post_json['commits'][0]['author']['email'],
post_json['user_username'] + '@elettra.eu', post_json['user_email']],
default_branch=post_json['project']['default_branch'])
else:
continue
# Assign the job to the builder with shortest queue length
idx = allbuilders[r.platform_id].index(min(allbuilders[r.platform_id],
key=lambda x:x.queue.qsize()))
logger.info("Assign building of " + r.name + " to " + allbuilders[r.platform_id][idx].name)
allbuilders[r.platform_id][idx].queue.put(job)
self.send_response(HTTPStatus.OK.value)
self.end_headers()
except Exception as e:
sendEmailAdmins("Receive new tag failed", str(e))
logger.error("Receive new tag failed: ", str(e))
session.rollback()
self.send_response(HTTPStatus.INTERNAL_SERVER_ERROR.value)
self.end_headers()
finally:
session.close()
def run(address, port, server_class=HTTPServer, handler_class=Server):
logger.info('Starting...')
server_address = (address, port)
httpd = server_class(server_address, handler_class)
try:
httpd.serve_forever()
except KeyboardInterrupt:
pass
httpd.server_close()
logger.info('Stopping...')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--db", type=str, help='Database URI to connect to', required=True)
parser.add_argument('--bind', type=str, default='localhost', help='IP Address or hostname to bind to')
parser.add_argument('--port', type=int, default=443, help='Port to listen to')
parser.add_argument("--store", type=str, default='/scratch/build/files-store/', help='Directory where store produced binaries')
parser.add_argument("--repo", type=str, default='/scratch/build/repositories/', help='Directory where checkout git repositories')
parser.add_argument("--smtpserver", type=str, default="smtp", help='Hostname of the SMTP server')
parser.add_argument("--smtpsender", type=str, default="noreply", help='Email sender')
parser.add_argument("--smtpdomain", type=str, default="elettra.eu", help='Email domain')
args = parser.parse_args()
print("Start inau-dispatcher using", args.db, "on interface", args.bind, "and port",
args.port, "file store directory", args.store, "repositories clone directory", args.repo,
"SMTP server", args.smtpserver, "SMTP sender", args.smtpsender, "SMTP domain", args.smtpdomain)
if os.getpgrp() == os.tcgetpgrp(sys.stdout.fileno()):
# Executed in foreground so redirect log to terminal and enable SQL echoing (Development)
logging.basicConfig(level=logging.INFO)
engine = create_engine(args.db, pool_pre_ping=True, echo=True)
else:
# Executed in background so redirect log to syslog and disable SQL echoing (Production)
syslog_handler = logging.handlers.SysLogHandler(address='/dev/log')
logging.basicConfig(level=logging.INFO, handlers=[syslog_handler])
engine = create_engine(args.db, pool_pre_ping=True, echo=False)
logger = logging.getLogger('inau-dispatcher')
Session.configure(bind=engine)
reconcile()
signal.signal(signal.SIGHUP, signalHandler)
if args.bind:
run(args.bind,args.port)
This diff is collapsed.
<!doctype html>
<html lang="en">
<head>
<!-- Required meta tags -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<!-- Bootstrap CSS -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css"
integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
<style>
body {
font-family: 'PT Sans Narrow', Arial, sans-serif;
font-size: 14px;
}
</style>
<title>INAU INstallazione AUtomatica</title>
</head>
<body>
<div class="container">
<div class="row">
<div class="col-12 center-block text-center">
<img src='http://www.elettra.eu/html/page/images/elettra.jpg'/>
<br><br><br><br>
<h4>INAU INstallazione AUtomatica</h4>
<br><br>
</div>
</div>
<div class="row">
<table class="table table-borderless">
<thead>
<tr>
{% for key, value in data[0].items() %}
<th scope="col">{{ key | e }}</th>
{% endfor %}
</tr>
</thead>
<tbody>
{% for row in data %}
<tr>
{% for key, value in row.items() %}
<td>{{ value | e }}</td>
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
</div>
<br>
<div class="row">
<div class="col-12 center-block text-center">
<small>
<address>
<strong>Sincrotrone Trieste S.C.p.A.</strong><br>
Strada Statale 14 - km 163,5 in AREA Science Park<br>
34149 Basovizza, Trieste ITALY<br>
Tel. +39 040 37581 - Fax. +39 040 9380902<br>
</address>
<small>
</div>
</div>
</div>
<!-- Optional JavaScript -->
<!-- jQuery first, then Popper.js, then Bootstrap JS -->
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"
integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"
integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous">
</script>
</body>
</html>