Examples¶
The scripts of the directory examples of the repository, with the
Application Default Credentials that the client libraries of Google find, of
any kind, as Pyjo.GCP.Credentials.Chain finds them.
Cloud Storage¶
A bucket of Cloud Storage as a website, with Pyjo.GCP.Client and the
calls of Pyjo.GCP.Call, whose requests the actions await in the event
loop of the server, without a client library of Google: the content of an
object, as it comes, of a call that forwards it to the client, with the
headers of its metadata, index.html of a folder, or the list of its
folders and objects.
uv run python examples/gcs.py --bucket mybucket daemon
"""
__DATA__
@@ folder.html.ep
<!DOCTYPE html>
<html>
<head><title>/<%= prefix %></title></head>
<body>
<h1>/<%= prefix %></h1>
<ul>
% if prefix:
<li><a href="../">../</a></li>
% end
% for name in names:
<li><a href="<%= url_for('object', name=prefix + name) %>"><%= name %></a></li>
% end
</ul>
</body>
</html>
"""
import Pyjo.Exception
import Pyjo.GCP.Call
import Pyjo.GCP.Client
import Pyjo.GCP.Credentials.Chain
import Pyjo.GCP.Credentials.Impersonated
import Pyjo.GCP.Error
import Pyjo.GetOpt
import Pyjo.IOLoop
import Pyjo.UserAgent
from Pyjo.Util import die, getopt, url_escape
from Pyjoyment.Lite import *
import os
import threading
# The options before the command, with the names of the gcloud CLI and the client libraries of Google, which set the
# environment variables that the application reads:
# python gcs.py --impersonate-service-account reader@myproject.iam.gserviceaccount.com --bucket mybucket daemon
getopt(
{
'bucket=s': Pyjo.GetOpt.env('GCS_BUCKET'),
'emulator-host=s': Pyjo.GetOpt.env('STORAGE_EMULATOR_HOST'),
'impersonate-service-account=s': Pyjo.GetOpt.env('CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT'),
},
pass_through=True,
require_order=True,
)
# The bucket, whose objects the application serves
bucket = os.environ.get('GCS_BUCKET') or die(
'Usage: gcs.py --bucket BUCKET [--impersonate-service-account EMAIL] [--emulator-host HOST] COMMAND [OPTIONS]'
)
root = '/storage/v1/b/' + url_escape(bucket)
# The token reads the objects only, of a service account or the metadata server, which take scopes; the one of
# gcloud auth application-default login has the scopes of the login
SCOPES = ['https://www.googleapis.com/auth/devstorage.read_only']
# The Application Default Credentials, as the client libraries of Google find them: of GOOGLE_APPLICATION_CREDENTIALS,
# of gcloud auth application-default login, of Workload Identity Federation, or of the metadata server of Compute
# Engine, GKE or Cloud Run; and a token of the service account that they impersonate, as gcloud does with
# --impersonate-service-account, which the client libraries do not read. The source keeps the default scope of the
# chain, cloud-platform, as its token calls the IAM Service Account Credentials API, and only the token of the target
# has the scope of the bucket
def credentials(ua):
if target := os.environ.get('CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT'):
source = Pyjo.GCP.Credentials.Chain.new(ua=ua)
return Pyjo.GCP.Credentials.Impersonated.new(source=source, target_principal=target, scopes=SCOPES, ua=ua)
return Pyjo.GCP.Credentials.Chain.new(scopes=SCOPES, ua=ua)
# A client of Cloud Storage for each event loop, as the connections of its user agent belong to one, of the endpoint of
# STORAGE_EMULATOR_HOST without a token, as the client libraries of Google take it, with or without its scheme, and
# without x-goog-user-project of the quota project of the credentials: Cloud Storage bills a request with it to that
# project, and gives Cache-Control: no-cache to its response then, not the one of the metadata of the object; a bucket
# of requester pays would need it
local = threading.local()
def gcs():
if not hasattr(local, 'gcs'):
ua = Pyjo.UserAgent.new()
local.gcs = Pyjo.GCP.Client.new(service='storage', credentials=credentials(ua), quota_project=False, ua=ua)
return local.gcs
# The client closes its connections once the server stops
@hook('after_server_stop')
def after_server_stop(server, app):
if hasattr(local, 'gcs'):
local.gcs.ua.close()
del local.gcs
# The path of an object of the XML API, whose media has the headers of the metadata of the object, where objects.get of
# the JSON API with alt=media gives Content-Disposition: attachment and Cache-Control: no-cache of its own; the name
# keeps its slashes
def path(name):
return '/' + url_escape(bucket) + '/' + url_escape(name, r'^A-Za-z0-9\-._~/')
# objects.list with the names only, as the resource of an object has its metadata
def list_objects(prefix, **query):
return Pyjo.GCP.Call.new(
'GET', root + '/o', query={'prefix': prefix, 'fields': 'items(name),prefixes,nextPageToken', **query}
)
# The headers of an object that the response has, as Cloud Storage gives them
HEADERS = (
'Cache-Control',
'Content-Disposition',
'Content-Encoding',
'Content-Language',
'Content-Length',
'Content-Type',
'ETag',
'Last-Modified',
)
# The media of an object, whose content goes to the client as it comes, each chunk once the client has read the one
# before, so that an object of any size is not kept in memory; the error of the XML API is not JSON, so the call has
# its status only
class GetObject(Pyjo.GCP.Call.object):
def __init__(self, c, name):
super().__init__('GET', path(name))
self.c = c
self.gone = False
def build_tx(self, ua, url):
tx = super().build_tx(ua, url)
tx.res.content.set(auto_upgrade=False, auto_decompress=False).once('body', lambda content: self.forward(tx))
return tx
def forward(self, tx):
if tx.res.code != 200:
return
c, res = self.c, tx.res
for name in HEADERS:
if (value := res.headers.header(name)) is not None:
c.res.headers.header(name, value)
if res.headers.content_length == '0':
return c.rendered()
source = Pyjo.IOLoop.stream(tx.connection)
# The content emits an empty chunk too, of the end of the headers that come before it, which would end the
# response
def read(content, chunk):
if not chunk:
return
source.stop()
c.write(chunk, lambda c: source.start())
res.content.unsubscribe('read').on('read', read)
# A client that goes away before the end of the object closes the connection to Cloud Storage, which reads no
# more
def finish(c):
if not res.is_finished:
self.gone = True
Pyjo.IOLoop.remove(tx.connection)
c.on('finish', finish)
def parse(self, tx):
return None
async def send_object(c, name):
call = GetObject(c, name)
try:
await gcs().call_p(call)
except Pyjo.Exception.object:
if not call.gone:
raise
# The folders and the objects under a prefix, of the pages of objects.list, as a bucket without the hierarchical
# namespace has no folders, only names with slashes, and a folder that the console makes, an empty object of the
# prefix, which the list leaves out
async def send_folder(c, prefix):
names, count, token = [], 0, None
while True:
data = await gcs().call_p(list_objects(prefix, delimiter='/', pageToken=token))
items, prefixes = data.get('items', []), data.get('prefixes', [])
count += len(items) + len(prefixes)
names += [p.removeprefix(prefix) for p in prefixes]
names += [i['name'].removeprefix(prefix) for i in items if i['name'] != prefix]
if (token := data.get('nextPageToken')) is None:
break
if prefix and not count:
return c.reply.not_found()
c.render('folder', prefix=prefix, names=sorted(names))
def not_found(e):
return e.code == 404
# An object, "index.html" or the list of a folder, for a name that ends with a slash
@get('/*name', {'name': ''}, 'object')
async def serve(c):
name = c.param('name')
if name and not name.endswith('/'):
try:
return await send_object(c, name)
except Pyjo.GCP.Error.object as e:
if not not_found(e):
raise
# A folder without its slash
data = await gcs().call_p(list_objects(f'{name}/', maxResults=1))
if data.get('items') or data.get('prefixes'):
return c.redirect_to('object', name=f'{name}/')
return c.reply.not_found()
try:
await send_object(c, f'{name}index.html')
except Pyjo.GCP.Error.object as e:
if not not_found(e):
raise
await send_folder(c, name)
app.start()
Gmail¶
A message of the standard input sent with the Gmail API, in the place of
sendmail, with its options that PHP, cron and the others give it, as the
user of the credentials, or the subject of the domain-wide delegation of a
service account, of a key or impersonated without one, and a call of
Pyjo.GCP.Call whose request has the message itself, of the media
upload of messages.send. With -bd, an SMTP server of
Pyjo.IOLoop, whose messages go to the Gmail API in its event loop.
uv run python examples/sendmail.py --subject noreply@example.com -t -i < message.eml
uv run python examples/sendmail.py --subject noreply@example.com -bd --listen 127.0.0.1:2525
import Pyjo.Exception
import Pyjo.GCP.Call
import Pyjo.GCP.Client
import Pyjo.GCP.Config
import Pyjo.GCP.Credentials.Chain
import Pyjo.GCP.Credentials.File
import Pyjo.GCP.Credentials.Impersonated
import Pyjo.GCP.Credentials.ServiceAccount
import Pyjo.GCP.Error
import Pyjo.IOLoop
import Pyjo.UserAgent
from Pyjo.Util import getenv
import email.parser
import email.utils
import re
import signal
import socket
import sys
# sendmail of the Gmail API: a message of the standard input, sent as the user of the credentials, with the options of
# sendmail that PHP, cron and the others give it, so that sendmail_path of php.ini can be:
# sendmail_path = "/usr/bin/python3 /opt/pyjo-gcp/examples/sendmail.py --subject noreply@example.com -t -i"
# or, with -bd of sendmail, an SMTP server of the Gmail API for the programs that send with SMTP
USAGE = """\
Usage: sendmail.py [--subject EMAIL] [--impersonate-service-account EMAIL] [-t] [-i] [-f SENDER] [-F NAME]
[RECIPIENT ...] < MESSAGE
sendmail.py [--subject EMAIL] [--impersonate-service-account EMAIL] -bd [--listen ADDRESS:PORT]
"""
# The exit codes of sysexits.h, as sendmail has them, of which PHP and the others tell a message that is not sent; not
# EX_TEMPFAIL of a message that sendmail keeps to send it later, which PHP takes for a message that is sent, as the
# example keeps none
EX_USAGE = 64
EX_UNAVAILABLE = 69
EX_NOPERM = 77
EX_CONFIG = 78
# The token sends messages only, as the user of the credentials, or of the subject of a service account of the
# domain-wide delegation of Google Workspace
SCOPES = ['https://www.googleapis.com/auth/gmail.send']
def fail(code, message):
print(f'sendmail.py: {message}', file=sys.stderr)
sys.exit(code)
# The options of sendmail, by hand, as Pyjo.GetOpt would take -i for an abbreviation of a long option: -t and -i of
# PHP, -f and -F of the sender, with the value in the option or after it, the others of sendmail ignored, as -oi of
# -i, -odi and -v of cron, and the long options of the example, or their environment variables
def options(argv):
opts = {
'subject': getenv('GMAIL_SUBJECT'),
'impersonate': getenv('CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT'),
'headers': False,
'dots': True,
'sender': None,
'name': None,
'recipients': [],
'daemon': False,
'listen': '127.0.0.1:2525',
}
args = list(argv)
while args:
arg = args.pop(0)
if arg == '--':
opts['recipients'] += args
break
if arg in ('-h', '--help'):
sys.stdout.write(USAGE)
sys.exit(0)
if m := re.fullmatch(r'--(subject|impersonate-service-account|listen)(?:=(.*))?', arg):
if (value := m[2]) is None:
value = args.pop(0) if args else fail(EX_USAGE, f'{arg} needs a value')
opts[{'impersonate-service-account': 'impersonate'}.get(m[1], m[1])] = value
elif arg.startswith('--'):
fail(EX_USAGE, f'unknown option {arg}\n{USAGE}')
elif arg == '-bd':
opts['daemon'] = True
elif arg == '-t':
opts['headers'] = True
elif arg in ('-i', '-oi'):
opts['dots'] = False
elif arg[:2] in ('-f', '-F', '-r'):
value = arg[2:] or (args.pop(0) if args else fail(EX_USAGE, f'{arg} needs a value'))
opts['name' if arg[:2] == '-F' else 'sender'] = value
elif arg.startswith('-'):
continue
else:
opts['recipients'].append(arg)
return opts
# The message of the standard input, which ends with a line of a dot without -i, as sendmail has it, with the line
# breaks of CRLF of RFC 5322, as PHP gives LF only
def read_message(dots):
lines = []
for line in sys.stdin.buffer:
line = line.rstrip(b'\r\n')
if dots and line == b'.':
break
lines.append(line)
return b'\r\n'.join(lines) + b'\r\n'
def addresses(headers, *names):
values = [value for name in names for value in headers.get_all(name, [])]
return {address.lower() for _, address in email.utils.getaddresses(values) if address}
# The recipients of the headers of a message, To, Cc and Bcc
def recipients_of(message):
head, _, _ = message.partition(b'\r\n\r\n')
return addresses(email.parser.BytesHeaderParser().parsebytes(head + b'\r\n\r\n'), 'To', 'Cc', 'Bcc')
# The headers that the message needs, before the ones that it has: From of the sender without one, and Bcc of the
# recipients of the envelope, of the arguments or of RCPT TO, that its headers do not have, as Gmail sends to the
# recipients of the headers, To, Cc and Bcc, which it leaves out of the message, and not to an envelope of its own
def complete(message, sender=None, name=None, recipients=()):
head, _, _ = message.partition(b'\r\n\r\n')
headers = email.parser.BytesHeaderParser().parsebytes(head + b'\r\n\r\n')
added = []
if headers.get('From') is None and sender:
added.append(f'From: {email.utils.formataddr((name or "", sender))}')
known = addresses(headers, 'To', 'Cc', 'Bcc')
if missing := [r for r in recipients if email.utils.parseaddr(r)[1].lower() not in known]:
added.append(f'Bcc: {", ".join(missing)}')
return ''.join(f'{header}\r\n' for header in added).encode() + message
# The credentials of the Gmail API: of the service account that the Application Default Credentials impersonate, with
# the subject of its domain-wide delegation, of signJwt, without a key; of the key of a service account of
# GOOGLE_APPLICATION_CREDENTIALS with the subject; or of the Application Default Credentials of a user that has the
# scope of Gmail of its login
def credentials(opts, ua):
if target := opts['impersonate']:
source = Pyjo.GCP.Credentials.Chain.new(ua=ua)
return Pyjo.GCP.Credentials.Impersonated.new(
source=source, target_principal=target, subject=opts['subject'], scopes=SCOPES, ua=ua
)
if not opts['subject']:
return Pyjo.GCP.Credentials.Chain.new(scopes=SCOPES, ua=ua)
config = Pyjo.GCP.Config.new()
path = config.credentials_file or fail(
EX_CONFIG, '--subject needs the key of a service account of GOOGLE_APPLICATION_CREDENTIALS, or'
' --impersonate-service-account'
) # fmt: skip
provider = Pyjo.GCP.Credentials.File.new(path=path, config=config, scopes=SCOPES, ua=ua).provider_of(
config.credentials_info(path), path
)
if not isinstance(provider, Pyjo.GCP.Credentials.ServiceAccount.object):
fail(EX_CONFIG, f'--subject needs the key of a service account: {path}')
provider.subject = opts['subject']
return provider
# messages.send of the media upload, of the whole message of RFC 822 up to 35 MB, which the JSON of raw with base64
# would make larger, as the user of the token, "me"
class SendMessage(Pyjo.GCP.Call.object):
def __init__(self, message):
super().__init__('POST', '/upload/gmail/v1/users/me/messages/send', query={'uploadType': 'media'})
self.message = message
def build_tx(self, ua, url):
url = Pyjo.GCP.Call.url_for(url, self.path, self.query)
return ua.build_tx(self.method, url, {**self.headers, 'Content-Type': 'message/rfc822'}, self.message)
def exit_code(e):
return EX_NOPERM if e.code in (401, 403) else EX_UNAVAILABLE
# The message of the standard input, of the recipients of the arguments, or of its headers with -t
def send(opts, gmail):
message = read_message(opts['dots'])
if not opts['headers'] and not opts['recipients']:
fail(EX_USAGE, f'no recipients, of the arguments or of -t\n{USAGE}')
if not opts['recipients'] and not recipients_of(message):
fail(EX_USAGE, 'no recipients in the To, Cc and Bcc headers of the message')
message = complete(message, opts['sender'], opts['name'], opts['recipients'])
try:
gmail.call(SendMessage(message))
except Pyjo.GCP.Error.object as e:
fail(exit_code(e), f'cannot send the message: {e.message}')
except Pyjo.Exception.object as e:
fail(EX_UNAVAILABLE, f'cannot send the message: {e.message}')
# The size of a message of the media upload of messages.send, 35 MB, which SIZE of EHLO tells the client
MAX_SIZE = 35 * 1024 * 1024
MAX_RECIPIENTS = 100
HOSTNAME = socket.gethostname()
# An SMTP session of RFC 5321, of a client of -bd, whose messages go to the Gmail API: a reply for each command, a
# message of DATA that ends with a line of a dot, whose dots of the client before a line are left out, and the commands
# that the client sends before a reply, of PIPELINING, kept in the buffer until the message before them is sent, as the
# Gmail API answers in the event loop of the server; there is no AUTH, so the server listens on the loopback by default
class Session:
def __init__(self, stream, gmail):
self.stream = stream
self.gmail = gmail
self.buffer = b''
self.busy = False
self.closed = False
self.reset()
stream.timeout = 300
stream.on('read', lambda stream, chunk: self.read(chunk))
stream.on('close', lambda stream: setattr(self, 'closed', True))
self.reply(220, f'{HOSTNAME} ESMTP sendmail.py of the Gmail API')
def reset(self):
self.sender = None
self.recipients = []
self.data = None
self.size = 0
def reply(self, code, *lines):
if self.closed:
return
for i, line in enumerate(lines):
self.stream.write(f'{code}{"-" if i < len(lines) - 1 else " "}{line}\r\n'.encode())
def read(self, chunk):
self.buffer += chunk
self.process()
def process(self):
while not self.busy and not self.closed and (end := self.buffer.find(b'\n')) != -1:
line, self.buffer = self.buffer[:end].rstrip(b'\r'), self.buffer[end + 1 :]
if self.data is None:
self.command(line.decode('utf-8', 'replace'))
elif line == b'.':
self.finish()
else:
self.size += len(line) + 2
if self.size <= MAX_SIZE:
self.data.append(line[1:] if line.startswith(b'.') else line)
if self.data is None and len(self.buffer) > 4096:
self.buffer = b''
self.reply(500, '5.5.6 Line too long')
def command(self, line):
verb, _, arg = line.partition(' ')
verb = verb.upper()
if verb == 'EHLO':
self.reset()
self.reply(250, HOSTNAME, f'SIZE {MAX_SIZE}', '8BITMIME', 'PIPELINING')
elif verb == 'HELO':
self.reset()
self.reply(250, HOSTNAME)
elif verb == 'MAIL':
if not (m := re.fullmatch(r'(?i)FROM:\s*<([^>]*)>(.*)', arg)):
return self.reply(501, '5.5.4 Syntax: MAIL FROM:<address>')
if self.sender is not None:
return self.reply(503, '5.5.1 Sender already given')
if (size := re.search(r'(?i)\bSIZE=(\d+)', m[2])) and int(size[1]) > MAX_SIZE:
return self.reply(552, '5.3.4 Message too big')
self.sender = m[1]
self.reply(250, '2.1.0 OK')
elif verb == 'RCPT':
if self.sender is None:
return self.reply(503, '5.5.1 MAIL first')
if not (m := re.fullmatch(r'(?i)TO:\s*<([^>]+)>.*', arg)):
return self.reply(501, '5.5.4 Syntax: RCPT TO:<address>')
if len(self.recipients) >= MAX_RECIPIENTS:
return self.reply(452, '4.5.3 Too many recipients')
self.recipients.append(m[1])
self.reply(250, '2.1.5 OK')
elif verb == 'DATA':
if not self.recipients:
return self.reply(503, '5.5.1 RCPT first')
self.data = []
self.reply(354, 'End data with <CR><LF>.<CR><LF>')
elif verb == 'RSET':
self.reset()
self.reply(250, '2.0.0 OK')
elif verb == 'NOOP':
self.reply(250, '2.0.0 OK')
elif verb == 'VRFY':
self.reply(252, '2.1.5 Cannot VRFY user')
elif verb == 'QUIT':
self.reply(221, '2.0.0 Bye')
self.stream.close_gracefully()
elif verb in ('STARTTLS', 'AUTH'):
self.reply(502, f'5.5.1 {verb} not implemented')
else:
self.reply(500, '5.5.2 Command not recognized')
# The message to the Gmail API, with the From of MAIL FROM and the recipients of RCPT TO, whose reply waits for the
# one of the API: a temporary error of 451, which the client tries again, and a permanent one of 550 or 554
def finish(self):
if self.size > MAX_SIZE:
self.reset()
return self.reply(552, '5.3.4 Message too big')
message = complete(b'\r\n'.join(self.data or []) + b'\r\n', self.sender, None, self.recipients)
self.busy = True
def sent(result):
self.reply(250, f'2.0.0 OK {result.get("id", "") if isinstance(result, dict) else ""}'.rstrip())
def failed(e):
if isinstance(e, Pyjo.GCP.Error.object) and e.code in (401, 403):
self.reply(550, f'5.7.1 {e.message}')
elif isinstance(e, Pyjo.GCP.Error.object) and e.code != 429 and e.code < 500:
self.reply(554, f'5.6.0 {e.message}')
else:
self.reply(451, f'4.3.0 {getattr(e, "message", e)}')
def done(*_):
self.reset()
self.busy = False
self.process()
self.gmail.call_p(SendMessage(message)).then(sent, failed).then(done, done)
# The SMTP server of -bd on the loopback by default, until SIGINT or SIGTERM stops it
def daemon(opts, gmail):
host, _, port = opts['listen'].rpartition(':')
if not port.isdigit():
fail(EX_USAGE, f'--listen of ADDRESS:PORT, not {opts["listen"]}')
try:
Pyjo.IOLoop.server(lambda loop, stream, id: Session(stream, gmail), address=host or '127.0.0.1', port=int(port))
except OSError as e:
fail(EX_CONFIG, f'cannot listen on {opts["listen"]}: {e.strerror or e}')
for number in (signal.SIGINT, signal.SIGTERM):
signal.signal(number, lambda *_: Pyjo.IOLoop.stop())
# A socket pair that Python writes to on a signal wakes the event loop up, as Pyjo.Server.Daemon does: Python
# repeats a select that a signal interrupts, so that a server without connections would not see its stop
waker, wakee = socket.socketpair()
waker.setblocking(False)
wakee.setblocking(False)
signal.set_wakeup_fd(waker.fileno())
reactor = Pyjo.IOLoop.singleton.reactor
reactor.io(wakee, lambda reactor, writable: wakee.recv(4096)).watch(wakee, True, False)
try:
Pyjo.IOLoop.start()
finally:
signal.set_wakeup_fd(-1)
reactor.remove(wakee)
waker.close()
wakee.close()
def main():
opts = options(sys.argv[1:])
ua = Pyjo.UserAgent.new()
gmail = Pyjo.GCP.Client.new(service='gmail', credentials=credentials(opts, ua), ua=ua)
try:
# The credentials first, of an error of the configuration, as a subject that the domain-wide delegation does not
# have, where an error of the call may be of now only
if not gmail.anonymous:
try:
gmail.credentials.credentials()
except Pyjo.Exception.object as e:
fail(EX_CONFIG, f'cannot get the credentials: {e.message}')
if opts['daemon']:
daemon(opts, gmail)
else:
send(opts, gmail)
finally:
ua.close()
if __name__ == '__main__':
main()