[RFC,CFH,sumo,26/47] cve-update-db-native: clean up JSON fetching

Message ID 311cfe9aa30c7d4a9476dec12769d2c4290f5a91.1573047194.git.mikko.rapeli@bmw.de
State New
Headers show
Series
  • Untitled series #24753
Related show

Commit Message

Mikko Rapeli Nov. 6, 2019, 3:37 p.m.
From: Ross Burton <ross.burton@intel.com>


Currently the code fetches the compressed JSON, writes it to a temporary file,
uncompresses that with gzip and passes the fake file object to update_db().

Instead, uncompress the gzip'd data in memory and pass the JSON directly to
update_db().

(From OE-Core rev: 9422745979256c442f533770203f62ec071c18fb)

Signed-off-by: Ross Burton <ross.burton@intel.com>

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

---
 meta/recipes-core/meta/cve-update-db-native.bb | 29 +++++++++++---------------
 1 file changed, 12 insertions(+), 17 deletions(-)

-- 
1.9.1

-- 
_______________________________________________
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core

Patch

diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb
index 6907197..a06b74a 100644
--- a/meta/recipes-core/meta/cve-update-db-native.bb
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -62,25 +62,20 @@  python do_populate_cve_db() {
         meta = c.fetchone()
         if not meta or meta[0] != last_modified:
             # Clear products table entries corresponding to current year
-            cve_year = 'CVE-' + str(year) + '%'
-            c.execute("delete from PRODUCTS where ID like ?", (cve_year,))
+            c.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,))
 
             # Update db with current year json file
-            req = urllib.request.Request(json_url)
-            if proxy:
-                req.set_proxy(proxy, 'https')
             try:
-                with urllib.request.urlopen(req, timeout=1) as r, \
-                     open(json_tmpfile, 'wb') as tmpfile:
-                    shutil.copyfileobj(r, tmpfile)
-            except:
+                req = urllib.request.Request(json_url)
+                if proxy:
+                    req.set_proxy(proxy, 'https')
+                with urllib.request.urlopen(req) as r:
+                    update_db(c, gzip.decompress(r.read()))
+                c.execute("insert or replace into META values (?, ?)", [year, last_modified])
+            except urllib.error.URLError as e:
                 cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
-                break
-
-            with gzip.open(json_tmpfile, 'rt') as jsonfile:
-                update_db(c, jsonfile)
-            c.execute("insert or replace into META values (?, ?)",
-                    [year, last_modified])
+                bb.warn("Cannot parse CVE data (%s), update failed" % e.reason)
+                return
 
         # Update success, set the date to cve_check file.
         if year == date.today().year:
@@ -143,9 +138,9 @@  def parse_node_and_insert(c, node, cveId):
 
     c.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator())
 
-def update_db(c, json_filename):
+def update_db(c, jsondata):
     import json
-    root = json.load(json_filename)
+    root = json.loads(jsondata)
 
     for elt in root['CVE_Items']:
         if not elt['impact']: