=== modified file 'linaro-image-indexer'
@@ -27,51 +27,93 @@
import bz2
import linaro_image_tools.fetch_image
-RELEASES_WWW_DOCUMENT_ROOT = "/srv/releases.linaro.org/www/platform/"
-RELEASE_URL = "http://releases.linaro.org/platform/"
+RELEASES_WWW_DOCUMENT_ROOT = "/srv/releases.linaro.org/www"
+RELEASE_URL = "http://releases.linaro.org/"
+OLD_RELEASES_WWW_DOCUMENT_ROOT = "/srv/releases.linaro.org/www/platform"
+OLD_RELEASE_URL = "http://releases.linaro.org/platform/"
SNAPSHOTS_WWW_DOCUMENT_ROOT = "/srv/snapshots.linaro.org/www/"
-SNAPSHOTS_URL = "http://snapshots.linaro.org/"
+SNAPSHOTS_URL = "http://snapshots.linaro.org/"
class ServerIndexer():
"""Create a database of files on the linaro image servers for use by image
creation tools."""
def reset(self):
- self.url_parse = {}
+ self.url_parse = []
def __init__(self):
self.reset()
self.db_file_name = "server_index"
self.db = linaro_image_tools.fetch_image.DB(self.db_file_name)
+ def regexp_list_matches_some(self, to_search, list):
+ assert len(list), "empty list passed"
+
+ for item in list:
+ if re.search(item, to_search):
+ return True
+
+ return False
+
+ def regexp_list_matches_all(self, to_search, list):
+ assert len(list), "empty list passed"
+
+ for item in list:
+ if not re.search(item, to_search):
+ return False
+
+ return True
+
def crawl(self):
self.db.set_url_parse_info(self.url_parse)
- logging.info(self.url_parse.items())
-
- for table, info in self.url_parse.items():
- logging.info(info["base_dir"], ":", info["base_url"], table,
- info["url_validator"], info["url_chunks"])
- self.go(info["base_dir"], info["base_url"], table)
- logging.info("")
-
- def go(self, root_dir_, root_url_, table_):
- for root, subFolders, files in os.walk( root_dir_ ):
-
- # --- Temporary hack to work around bug:
- # https://bugs.launchpad.net/linaro-image-tools/+bug/816015
- # For the moment we just index platform == 11.05-daily when
- # indexing the snapshots server.
- if re.search("11.05-daily", root) or re.search("release", table_):
- for file in files:
- if(re.search('\.gz$', file)):
- # Construct a URL to the file and save in the database
- relative_location = re.sub(root_dir_, "",
- os.path.join(root, file))
- url = urlparse.urljoin(root_url_, relative_location)
- url = urlparse.urljoin(url, file)
-
- if not re.search('/leb-panda/', url):
- logging.info(url)
- self.db.record_url(url, table_)
+ logging.getLogger("linaro_image_tools").info(self.url_parse)
+
+ for index in range(len(self.url_parse)):
+ info = self.url_parse[index]
+ table = info["table"]
+
+ logging.getLogger("linaro_image_tools").info("%s %s %s %s %s" % \
+ (info["base_dir"], info["base_url"], table,
+ info["url_validator"], info["url_chunks"]))
+
+ self.go(info, table, index)
+
+ logging.getLogger("linaro_image_tools").info("")
+
+
+ def go(self, info, table, index):
+ root_url = info["base_url"]
+ root_dir = info["base_dir"]
+
+ for root, subFolders, files in os.walk( root_dir ):
+ for file in files:
+ relative_location = re.sub(root_dir, "",
+ os.path.join(root, file))
+ relative_location = relative_location.lstrip("/")
+
+ to_match = info["url_validator"][0]
+ not_match = info["url_validator"][1]
+
+ url = urlparse.urljoin(root_url, relative_location)
+ url = urlparse.urljoin(url, file)
+
+ to_match_ok = False
+ if len(to_match) == 0:
+ to_match_ok = True
+ if len(to_match) and self.regexp_list_matches_all(
+ relative_location, to_match):
+ to_match_ok = True
+
+ not_match_ok = True
+ if len(not_match) and self.regexp_list_matches_some(
+ relative_location, not_match):
+ not_match_ok = False
+
+ if( not (to_match_ok and not_match_ok)
+ or not re.search("\.gz$", file)):
+ continue # URL doesn't match the validator. Ignore.
+
+ logging.getLogger("linaro_image_tools").info(url)
+ self.db.record_url(url, index)
self.dump()
@@ -88,22 +130,25 @@
bz2_db_file.close()
def add_directory_parse_list(self,
- base_dir_,
- base_url_,
- url_validator_,
- id_,
- url_chunks_):
+ base_dir,
+ base_url,
+ url_validator,
+ db_columns,
+ table,
+ url_chunks):
- if(not id_ in self.url_parse):
- self.url_parse[id_] = {"base_dir": base_dir_,
- "base_url": base_url_,
- "url_validator": url_validator_,
- "url_chunks": url_chunks_}
- logging.info(self.url_parse[id_]["base_dir"])
+ if not id in self.url_parse:
+ self.url_parse.append({"base_dir": base_dir,
+ "base_url": base_url,
+ "url_validator": url_validator,
+ "db_columns": db_columns,
+ "url_chunks": url_chunks,
+ "table": table})
+ logging.getLogger("linaro_image_tools").info(base_dir)
# Construct data needed to create the table
items = []
- for item in url_chunks_:
+ for item in url_chunks:
if(item != ""):
# If the entry is a tuple, it indicates it is of the
# form name, regexp
@@ -112,7 +157,7 @@
else:
items.append(item)
- self.db.create_table_with_url_text_items(id_, items)
+ self.db.create_table_with_name_columns(table, db_columns)
def clean_removed_urls_from_db(self):
self.db.clean_removed_urls_from_db()
@@ -120,40 +165,72 @@
if __name__ == '__main__':
crawler = ServerIndexer()
- # The use of a zero width assertion here to look for links that don't
- # contain /hwpacks/ is a bit scary and could be replaced by a tuple of
- # (False, r"hwpacks"), where the first parameter could indicate that we
- # want the regexp to fail if we are to use the URL. May be a bit nicer.
-
- #http://releases.linaro.org/platform/linaro-m/plasma/final/
- crawler.add_directory_parse_list(RELEASES_WWW_DOCUMENT_ROOT,
- RELEASE_URL,
- r"^((?!hwpack).)*$",
- "release_binaries",
- ["platform", "image", "build"])
-
- #http://releases.linaro.org/platform/linaro-m/hwpacks/final/hwpack_linaro-bsp-omap4_20101109-1_armel_unsupported.tar.gz
- crawler.add_directory_parse_list(RELEASES_WWW_DOCUMENT_ROOT,
- RELEASE_URL,
- r"/hwpacks/",
- "release_hwpacks",
- ["platform", "", "build",
- ("hardware", r"hwpack_linaro-(.*?)_")])
-
- #http://snapshots.linaro.org/11.05-daily/linaro-alip/20110420/0/images/tar/
+ ch = logging.StreamHandler()
+ ch.setLevel(logging.CRITICAL)
+ formatter = logging.Formatter("%(message)s")
+ ch.setFormatter(formatter)
+ logger = logging.getLogger("linaro_image_tools")
+ logger.setLevel(logging.CRITICAL)
+ logger.addHandler(ch)
+
+ #linaro-n/ubuntu-desktop/11.09
+ crawler.add_directory_parse_list(OLD_RELEASES_WWW_DOCUMENT_ROOT,
+ OLD_RELEASE_URL,
+ ([], ["platform/", "old/", "hwpack",
+ "alpha", "beta", "final", "leb",
+ "leb", "release-candidate"]),
+ ["platform", "image", "build=final"],
+ "release_binaries",
+ ["", "image", "platform"])
+
+ #linaro-n/hwpacks/11.09
+ crawler.add_directory_parse_list(OLD_RELEASES_WWW_DOCUMENT_ROOT,
+ OLD_RELEASE_URL,
+ (["/hwpacks/"],
+ ["alpha", "beta", "final", "leb",
+ "release-candidate"]),
+ ["platform", "hardware", "build=final"],
+ "release_hwpacks",
+ ["", "", "platform",
+ ("hardware", r"hwpack_linaro-(.*?)_")])
+
+ # 11.10/ubuntu/oneiric-images/ubuntu-desktop/
+ # NOT images/...
+ crawler.add_directory_parse_list(RELEASES_WWW_DOCUMENT_ROOT,
+ RELEASE_URL,
+ (["\d+\.\d+", "ubuntu", "oneiric-images"],
+ ["latest/", "platform/", "old/",
+ "hwpack", "^images/"]),
+ ["platform", "image", "build=final"],
+ "release_binaries",
+ ["platform", "", "", "image"])
+
+ # 11.10/ubuntu/oneiric-hwpacks/
+ crawler.add_directory_parse_list(RELEASES_WWW_DOCUMENT_ROOT,
+ RELEASE_URL,
+ (["\d+\.\d+", "ubuntu", "oneiric-hwpacks"],
+ ["latest/", "platform/", "old/",
+ "^images/"]),
+ ["platform", "hardware", "build=final"],
+ "release_hwpacks",
+ ["platform", "", "",
+ ("hardware", r"hwpack_linaro-(.*?)_")])
+
+ #oneiric/linaro-o-alip/20111026/0/images/tar/
crawler.add_directory_parse_list(SNAPSHOTS_WWW_DOCUMENT_ROOT,
SNAPSHOTS_URL,
- r"^((?!hwpack).)*$",
+ (["^oneiric/"], ["/hwpack"]),
+ ["platform", "image", "date", "build"],
"snapshot_binaries",
["platform", "image", "date", "build"])
- #http://snapshots.linaro.org/11.05-daily/linaro-hwpacks/omap3/20110420/0/images/hwpack/
+ #oneiric/lt-panda-oneiric/20111026/0/images/hwpack/
crawler.add_directory_parse_list(SNAPSHOTS_WWW_DOCUMENT_ROOT,
SNAPSHOTS_URL,
- r"/hwpack/",
+ (["^oneiric/", "/hwpack"], []),
+ ["platform", "hardware", "date", "build"],
"snapshot_hwpacks",
- ["platform", "", "hardware", "date",
- "build"])
+ ["platform", "hardware", "date", "build"])
crawler.crawl()
crawler.clean_removed_urls_from_db()
=== modified file 'linaro_image_tools/fetch_image.py'
@@ -36,7 +36,6 @@
import threading
import subprocess
import utils
-import xdg.BaseDirectory as xdgBaseDir
QEMU = "qemu"
HARDWARE = "hardware"
@@ -530,6 +529,10 @@
"""Downloads files and creates images from them by calling
linaro-media-create"""
def __init__(self):
+ # Import xdg here so it isn't required to index the server.
+ # (package not installed)
+ import xdg.BaseDirectory as xdgBaseDir
+
self.datadir = os.path.join(xdgBaseDir.xdg_data_home,
"linaro",
"image-tools",
@@ -916,31 +919,29 @@
def set_url_parse_info(self, url_parse):
self.url_parse = url_parse
- def record_url(self, url, table):
+ def record_url(self, url, index):
"""Check to see if the record exists in the index, if not, add it"""
- assert self.url_parse[table]["base_url"] != None, ("Can not match the "
+ assert self.url_parse[index]["base_url"], ("Can not match the "
"URL received (%s) to an entry provided by add_url_parse_list",
url)
- assert re.search('^' + self.url_parse[table]["base_url"], url)
-
- if(not re.search(self.url_parse[table]["url_validator"], url)):
- #Make sure that the URL matches the validator
- return
-
- logging.info("Recording URL", url)
+ assert re.search('^' + self.url_parse[index]["base_url"], url), (
+ "Base url is not part of the url to record.")
+
+ logging.info("Recording URL %s %d", url, index)
assert url not in self.touched_urls, ("URLs expected to only be added "
"to 1 place\n" + url)
self.touched_urls[url] = True
+ table = self.url_parse[index]["table"]
# Do not add the record if it already exists
self.c.execute("select url from " + table + " where url == ?", (url,))
- if(self.c.fetchone()):
+ if self.c.fetchone():
return
- url_match = re.search(self.url_parse[table]["base_url"] + r"(.*)$",
+ url_match = re.search(self.url_parse[index]["base_url"] + r"(.*)$",
url)
url_chunks = url_match.group(1).lstrip('/').encode('ascii').split('/')
# url_chunks now contains all parts of the url, split on /,
@@ -949,29 +950,39 @@
# We now construct an SQL command to insert the index data into the
# database using the information we have.
- # Work out how many values we will insert into the database
+ sqlcmd = "INSERT INTO " + table + " ("
length = 0
- for name in self.url_parse[table]["url_chunks"]:
- if(name != ""):
- length += 1
-
- sqlcmd = "insert into " + table + " values ("
+ for name in (self.url_parse[index]["url_chunks"] + ["url"]):
+ if name != "":
+ if isinstance(name, tuple):
+ name = name[0]
+ sqlcmd += name + ", "
+ length += 1
+
+ # Handle fixed value columns
+ for name in self.url_parse[index]["db_columns"]:
+ name_search = re.search("(\w+)=(.*)", name)
+ if name_search:
+ sqlcmd += name_search.group(1) + ", "
+ length += 1
+
+ sqlcmd = sqlcmd.rstrip(", ") # get rid of unwanted space & comma
+ sqlcmd += ") VALUES ("
# Add the appropriate number of ?s (+1 is so we have room for url)
- sqlcmd += "".join(["?, " for x in range(length + 1)])
- sqlcmd = sqlcmd.rstrip(" ") # get rid of unwanted space
- sqlcmd = sqlcmd.rstrip(",") # get rid of unwanted comma
+ sqlcmd += "".join(["?, " for x in range(length)])
+ sqlcmd = sqlcmd.rstrip(", ") # get rid of unwanted space and comma
sqlcmd += ")"
# Get the parameters from the URL to record in the SQL database
sqlparams = []
chunk_index = 0
- for name in self.url_parse[table]["url_chunks"]:
+ for name in self.url_parse[index]["url_chunks"]:
# If this part of the URL isn't a parameter, don't insert it
- if(name != ""):
+ if name != "":
# If the entry is a tuple, it indicates it is of the form
# name, regexp
- if(isinstance(name, tuple)):
+ if isinstance(name, tuple):
# use stored regexp to extract data for the database
match = re.search(name[1], url_chunks[chunk_index])
assert match, ("Unable to match regexp to string ",
@@ -985,6 +996,13 @@
sqlparams.append(url)
+ # Handle fixed value columns
+ for name in self.url_parse[index]["db_columns"]:
+ name_search = re.search("(\w+)=(.*)", name)
+ if name_search:
+ sqlparams.append(name_search.group(2))
+
+ logging.info("{0}: {1}".format(sqlcmd, sqlparams))
self.c.execute(sqlcmd, tuple(sqlparams))
def commit(self):
@@ -995,11 +1013,13 @@
self.commit()
self.database.close()
- def create_table_with_url_text_items(self, table, items):
+ def create_table_with_name_columns(self, table, items):
cmd = "create table if not exists "
cmd += table + " ("
+ # Handle fixed items (kept in because a field can no longer be derived)
for item in items:
+ item = re.sub("=.*", "", item)
cmd += item + " TEXT, "
cmd += "url TEXT)"
@@ -1031,7 +1051,9 @@
def clean_removed_urls_from_db(self):
self.c = self.database.cursor()
- for table, info in self.url_parse.items():
+ for info in self.url_parse:
+ table = info["table"]
+
self.c.execute("select url from " + table)
to_delete = []
=== modified file 'linaro_image_tools/fetch_image_settings.yaml'
@@ -10,12 +10,21 @@
platform: Which Linaro platform to build an image from. Specify 'snapshot' to
use a snapshot rather than a release.
developer: Developer Tools
+ o-developer: Developer Tools
graphics: Graphics
+ o-graphics: Graphics
multimedia: Multimedia
+ o-multimedia: Multimedia
nano: Nano
+ o-nano: Nano
ubuntu-desktop: 'LEB: Linaro Ubuntu Desktop'
+ o-ubuntu-desktop: 'LEB: Linaro Ubuntu Desktop'
ubuntu-desktop::long: Linux for humans on low power machines
ubuntu-desktop::release_note: Shiny!
+ o-alip: ARM Linux Internet Platform
+ alip: ARM Linux Internet Platform
+ server: Server
+ o-server: Server
hwpack-descriptions:
omap3: Basic support for Beagle boards
omap3-x11-base: Includes support for 3D acceleration
@@ -63,9 +72,12 @@
beagle:
- omap3
- omap3-x11-base
+ - omap3-oneiric
+ - omap3-x11-base-oneiric
igep:
- igep
+ - igep-oneiric
panda:
- panda
@@ -73,30 +85,41 @@
- bsp-omap4
- lt-panda-x11-base-natty
- lt-panda
+ - panda-oneiric
+ - panda-x11-base-oneiric
+ - lt-panda-x11-base-oneiric
+ - lt-panda-oneiric
vexpress:
- vexpress
+ - vexpress-oneiric
+ - lt-vexpress-a9-oneiric
ux500:
- bsp-ux500
efikamx:
- efikamx
+ - efikamx-oneiric
efikasb:
- efikamx
+ - efikamx-oneiric
mx51evk:
- imx51
+ - imx51-oneiric
mx53loco:
- lt-mx5
+ - lt-mx5-oneiric
u8500:
- lt-u8500
overo:
- overo
+ - overo-oneiric
smdkv310:
- lt-s5pv310
@@ -104,9 +127,13 @@
origen:
- lt-origen
+ - lt-origen-oneiric
snowball_sd:
- lt-snowball
+ - lt-snowball-v2
+ - lt-snowball-v3-oneiric
+ - lt-snowball-v2-oneiric
image:
- alip
@@ -135,6 +162,8 @@
- final
'11.09':
- final
+ '11.10':
+ - final
platform: linaro-n
repository: release