FM / sources

/usr/lib/python3.6/site-packages/cloudinit/sources/

/usr/lib/python3.6/site-packages/cloudinit/sources

Name	Size	Mode	Actions
azure/	-	0755	rm
helpers/	-	0755	rm
__pycache__/	-	0755	rm
DataSourceAkamai.py	12969	0644	edit dl rm
DataSourceAliYun.py	3037	0644	edit dl rm
DataSourceAltCloud.py	8423	0644	edit dl rm
DataSourceAzure.py	74697	0644	edit dl rm
DataSourceBigstep.py	1941	0644	edit dl rm
DataSourceCloudSigma.py	3956	0644	edit dl rm
DataSourceCloudStack.py	8281	0644	edit dl rm
DataSourceConfigDrive.py	11230	0644	edit dl rm
DataSourceDigitalOcean.py	4237	0644	edit dl rm
DataSourceEc2.py	36608	0644	edit dl rm
DataSourceExoscale.py	9000	0644	edit dl rm
DataSourceGCE.py	14150	0644	edit dl rm
DataSourceHetzner.py	5442	0644	edit dl rm
DataSourceIBMCloud.py	14371	0644	edit dl rm
DataSourceLXD.py	17463	0644	edit dl rm
DataSourceMAAS.py	15488	0644	edit dl rm
DataSourceNoCloud.py	13308	0644	edit dl rm
DataSourceNone.py	1370	0644	edit dl rm
DataSourceNWCS.py	4481	0644	edit dl rm
DataSourceOpenNebula.py	16143	0644	edit dl rm
DataSourceOpenStack.py	10580	0644	edit dl rm
DataSourceOracle.py	14936	0644	edit dl rm
DataSourceOVF.py	12963	0644	edit dl rm
DataSourceRbxCloud.py	8026	0644	edit dl rm
DataSourceScaleway.py	15796	0644	edit dl rm
DataSourceSmartOS.py	34785	0644	edit dl rm
DataSourceUpCloud.py	5609	0644	edit dl rm
DataSourceVMware.py	36909	0644	edit dl rm
DataSourceVultr.py	4585	0644	edit dl rm
__init__.py	44017	0644	edit dl rm

Edit: /usr/lib/python3.6/site-packages/cloudinit/sources/DataSourceMAAS.py (15488B)

# Copyright (C) 2012 Canonical Ltd.
# Copyright (C) 2012 Yahoo! Inc.
#
# Author: Scott Moser <scott.moser@canonical.com>
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
#
# This file is part of cloud-init. See LICENSE file for license information.

import hashlib
import logging
import os
import time

from cloudinit import sources, url_helper, util
from cloudinit.net.cmdline import KlibcNetworkConfigSource
from cloudinit.sources import NetworkConfigSource

LOG = logging.getLogger(__name__)
MD_VERSION = "2012-03-01"

DS_FIELDS = [
    # remote path, location in dictionary, binary data?, optional?
    ("meta-data/instance-id", "meta-data/instance-id", False, False),
    ("meta-data/local-hostname", "meta-data/local-hostname", False, False),
    ("meta-data/public-keys", "meta-data/public-keys", False, True),
    ("meta-data/vendor-data", "vendor-data", True, True),
    ("user-data", "user-data", True, True),
]

class DataSourceMAAS(sources.DataSource):
    """
    DataSourceMAAS reads instance information from MAAS.
    Given a config metadata_url, and oauth tokens, it expects to find
    files under the root named:
      instance-id
      user-data
      hostname
      vendor-data
    """

dsname = "MAAS"
    id_hash = None
    _oauth_helper = None

def __init__(self, sys_cfg, distro, paths):
        sources.DataSource.__init__(self, sys_cfg, distro, paths)
        self.base_url = None
        self.seed_dir = os.path.join(paths.seed_dir, "maas")
        self.id_hash = get_id_from_ds_cfg(self.ds_cfg)

@property
    def oauth_helper(self):
        if not self._oauth_helper:
            self._oauth_helper = get_oauth_helper(self.ds_cfg)
        return self._oauth_helper

def __str__(self):
        root = sources.DataSource.__str__(self)
        return "%s [%s]" % (root, self.base_url)

def _get_data(self):
        mcfg = self.ds_cfg

try:
            self._set_data(self.seed_dir, read_maas_seed_dir(self.seed_dir))
            return True
        except MAASSeedDirNone:
            pass
        except MAASSeedDirMalformed as exc:
            LOG.warning("%s was malformed: %s", self.seed_dir, exc)
            raise

# If there is no metadata_url, then we're not configured
        url = mcfg.get("metadata_url", None)
        if not url:
            return False

try:
            # doing this here actually has a side affect of
            # getting oauth time-fix in place.  As no where else would
            # retry by default, so even if we could fix the timestamp
            # we would not.
            if not self.wait_for_metadata_service(url):
                return False

self._set_data(
                url,
                read_maas_seed_url(
                    url,
                    read_file_or_url=self.oauth_helper.readurl,
                    paths=self.paths,
                    retries=1,
                ),
            )
            return True
        except Exception:
            util.logexc(LOG, "Failed fetching metadata from url %s", url)
            return False

def _set_data(self, url, data):
        # takes a url for base_url and a tuple of userdata, metadata, vd.
        self.base_url = url
        ud, md, vd = data
        self.userdata_raw = ud
        self.metadata = md
        self.vendordata_pure = vd
        if vd:
            try:
                self.vendordata_raw = sources.convert_vendordata(vd)
            except ValueError as e:
                LOG.warning("Invalid content in vendor-data: %s", e)
                self.vendordata_raw = None

def _get_subplatform(self):
        """Return the subplatform metadata source details."""
        return "seed-dir (%s)" % self.base_url

def wait_for_metadata_service(self, url):
        mcfg = self.ds_cfg
        max_wait = 120
        try:
            max_wait = int(mcfg.get("max_wait", max_wait))
        except Exception:
            util.logexc(LOG, "Failed to get max wait. using %s", max_wait)

if max_wait == 0:
            return False

timeout = 50
        try:
            if timeout in mcfg:
                timeout = int(mcfg.get("timeout", timeout))
        except Exception:
            LOG.warning("Failed to get timeout, using %s", timeout)

starttime = time.time()
        if url.endswith("/"):
            url = url[:-1]
        check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
        urls = [check_url]
        url, _response = self.oauth_helper.wait_for_url(
            urls=urls, max_wait=max_wait, timeout=timeout
        )

if url:
            LOG.debug("Using metadata source: '%s'", url)
        else:
            LOG.critical(
                "Giving up on md from %s after %i seconds",
                urls,
                int(time.time() - starttime),
            )

return bool(url)

def check_instance_id(self, sys_cfg):
        """locally check if the current system is the same instance.

MAAS doesn't provide a real instance-id, and if it did, it is
        still only available over the network.  We need to check based
        only on local resources.  So compute a hash based on Oauth tokens."""
        if self.id_hash is None:
            return False
        ncfg = util.get_cfg_by_path(sys_cfg, ("datasource", self.dsname), {})
        return self.id_hash == get_id_from_ds_cfg(ncfg)

class DataSourceMAASLocal(DataSourceMAAS):
    network_config_sources = (
        NetworkConfigSource.CMD_LINE,
        NetworkConfigSource.SYSTEM_CFG,
        NetworkConfigSource.DS,
        NetworkConfigSource.INITRAMFS,
    )

def _get_data(self):
        if not KlibcNetworkConfigSource().is_applicable():
            # We booted from disk. Initramfs didn't bring up a network, so
            # nothing to do. Wait until network timeframe to run _get_data()
            LOG.debug("No initramfs applicable config")
            return False
        LOG.debug("Found initramfs applicable config")
        return super()._get_data()

def get_oauth_helper(cfg):
    """Return an oauth helper instance for values in cfg.

@raises ValueError from OauthUrlHelper if some required fields have
            true-ish values but others do not."""
    keys = ("consumer_key", "consumer_secret", "token_key", "token_secret")
    kwargs = dict([(r, cfg.get(r)) for r in keys])
    return url_helper.OauthUrlHelper(**kwargs)

def get_id_from_ds_cfg(ds_cfg):
    """Given a config, generate a unique identifier for this node."""
    fields = ("consumer_key", "token_key", "token_secret")
    idstr = "\0".join([ds_cfg.get(f, "") for f in fields])
    # store the encoding version as part of the hash in the event
    # that it ever changed we can compute older versions.
    return "v1:" + hashlib.sha256(idstr.encode("utf-8")).hexdigest()

def read_maas_seed_dir(seed_d):
    if seed_d.startswith("file://"):
        seed_d = seed_d[7:]
    if not os.path.isdir(seed_d) or len(os.listdir(seed_d)) == 0:
        raise MAASSeedDirNone("%s: not a directory")

# seed_dir looks in seed_dir, not seed_dir/VERSION
    return read_maas_seed_url("file://%s" % seed_d, version=None)

def read_maas_seed_url(
    seed_url,
    read_file_or_url=None,
    timeout=None,
    version=MD_VERSION,
    paths=None,
    retries=None,
):
    """
    Read the maas datasource at seed_url.
      read_file_or_url is a method that should provide an interface
      like util.read_file_or_url

Expected format of seed_url is are the following files:
      * <seed_url>/<version>/meta-data/instance-id
      * <seed_url>/<version>/meta-data/local-hostname
      * <seed_url>/<version>/user-data
    If version is None, then <version>/ will not be used.
    """
    if read_file_or_url is None:
        read_file_or_url = url_helper.read_file_or_url

if seed_url.endswith("/"):
        seed_url = seed_url[:-1]

md = {}
    for path, _dictname, binary, optional in DS_FIELDS:
        if version is None:
            url = "%s/%s" % (seed_url, path)
        else:
            url = "%s/%s/%s" % (seed_url, version, path)
        try:
            ssl_details = util.fetch_ssl_details(paths)
            resp = read_file_or_url(
                url, retries=retries, timeout=timeout, ssl_details=ssl_details
            )
            if resp.ok():
                if binary:
                    md[path] = resp.contents
                else:
                    md[path] = util.decode_binary(resp.contents)
            else:
                LOG.warning(
                    "Fetching from %s resulted in an invalid http code %s",
                    url,
                    resp.code,
                )
        except url_helper.UrlError as e:
            if e.code == 404 and not optional:
                raise MAASSeedDirMalformed(
                    "Missing required %s: %s" % (path, e)
                ) from e
            elif e.code != 404:
                raise e

return check_seed_contents(md, seed_url)

def check_seed_contents(content, seed):
    """Validate if dictionary content valid as a return for a datasource.
    Either return a (userdata, metadata, vendordata) tuple or
    Raise MAASSeedDirMalformed or MAASSeedDirNone
    """
    ret = {}
    missing = []
    for spath, dpath, _binary, optional in DS_FIELDS:
        if spath not in content:
            if not optional:
                missing.append(spath)
            continue

if "/" in dpath:
            top, _, p = dpath.partition("/")
            if top not in ret:
                ret[top] = {}
            ret[top][p] = content[spath]
        else:
            ret[dpath] = content[spath]

if len(ret) == 0:
        raise MAASSeedDirNone("%s: no data files found" % seed)

if missing:
        raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing))

vd_data = None
    if ret.get("vendor-data"):
        err = object()
        vd_data = util.load_yaml(
            ret.get("vendor-data"), default=err, allowed=(object)
        )
        if vd_data is err:
            raise MAASSeedDirMalformed("vendor-data was not loadable as yaml.")

return ret.get("user-data"), ret.get("meta-data"), vd_data

class MAASSeedDirNone(Exception):
    pass

class MAASSeedDirMalformed(Exception):
    pass

# Used to match classes to dependencies
datasources = [
    (DataSourceMAAS, (sources.DEP_FILESYSTEM,)),
    (DataSourceMAAS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
]

# Return a list of data sources that match this set of dependencies
def get_datasource_list(depends):
    return sources.list_from_depends(depends, datasources)

if __name__ == "__main__":

def main():
        """
        Call with single argument of directory or http or https url.
        If url is given additional arguments are allowed, which will be
        interpreted as consumer_key, token_key, token_secret, consumer_secret
        """
        import argparse
        import pprint
        import sys

parser = argparse.ArgumentParser(description="Interact with MAAS DS")
        parser.add_argument(
            "--config",
            metavar="file",
            help="specify DS config file",
            default=None,
        )
        parser.add_argument(
            "--ckey",
            metavar="key",
            help="the consumer key to auth with",
            default=None,
        )
        parser.add_argument(
            "--tkey",
            metavar="key",
            help="the token key to auth with",
            default=None,
        )
        parser.add_argument(
            "--csec",
            metavar="secret",
            help="the consumer secret (likely '')",
            default="",
        )
        parser.add_argument(
            "--tsec",
            metavar="secret",
            help="the token secret to auth with",
            default=None,
        )
        parser.add_argument(
            "--apiver",
            metavar="version",
            help="the apiver to use ( can be used)",
            default=MD_VERSION,
        )

subcmds = parser.add_subparsers(title="subcommands", dest="subcmd")
        for (name, help) in (
            ("crawl", "crawl the datasource"),
            ("get", "do a single GET of provided url"),
            ("check-seed", "read and verify seed at url"),
        ):
            p = subcmds.add_parser(name, help=help)
            p.add_argument(
                "url", help="the datasource url", nargs="?", default=None
            )

args = parser.parse_args()

creds = {
            "consumer_key": args.ckey,
            "token_key": args.tkey,
            "token_secret": args.tsec,
            "consumer_secret": args.csec,
        }

if args.config is None:
            for fname in ("91_kernel_cmdline_url", "90_dpkg_maas"):
                fpath = "/etc/cloud/cloud.cfg.d/" + fname + ".cfg"
                if os.path.exists(fpath) and os.access(fpath, os.R_OK):
                    sys.stderr.write("Used config in %s.\n" % fpath)
                    args.config = fpath

if args.config:
            cfg = util.read_conf(args.config)
            if "datasource" in cfg:
                cfg = cfg["datasource"]["MAAS"]
            for key in creds.keys():
                if key in cfg and creds[key] is None:
                    creds[key] = cfg[key]
            if args.url is None and "metadata_url" in cfg:
                args.url = cfg["metadata_url"]

if args.url is None:
            sys.stderr.write("Must provide a url or a config with url.\n")
            sys.exit(1)

oauth_helper = get_oauth_helper(creds)

def geturl(url):
            # the retry is to ensure that oauth timestamp gets fixed
            return oauth_helper.readurl(url, retries=1).contents

def printurl(url):
            print("== %s ==\n%s\n" % (url, geturl(url).decode()))

def crawl(url):
            if url.endswith("/"):
                for line in geturl(url).decode().splitlines():
                    if line.endswith("/"):
                        crawl("%s%s" % (url, line))
                    elif line == "meta-data":
                        # meta-data is a dir, it *should* end in a /
                        crawl("%s%s" % (url, "meta-data/"))
                    else:
                        printurl("%s%s" % (url, line))
            else:
                printurl(url)

if args.subcmd == "check-seed":
            sys.stderr.write("Checking seed at %s\n" % args.url)
            readurl = oauth_helper.readurl
            if args.url[0] == "/" or args.url.startswith("file://"):
                (userdata, metadata, vd) = read_maas_seed_dir(args.url)
            else:
                (userdata, metadata, vd) = read_maas_seed_url(
                    args.url,
                    version=args.apiver,
                    read_file_or_url=readurl,
                    retries=2,
                )
            print("=== user-data ===")
            print("N/A" if userdata is None else userdata.decode())
            print("=== meta-data ===")
            pprint.pprint(metadata)
            print("=== vendor-data ===")
            pprint.pprint("N/A" if vd is None else vd)

elif args.subcmd == "get":
            printurl(args.url)

elif args.subcmd == "crawl":
            if not args.url.endswith("/"):
                args.url = "%s/" % args.url
            crawl(args.url)

main()

cmd: