[3/6] cve-update-db-native: clean up proxy handling

Message ID 20191118164647.29409-3-ross.burton@intel.com
State Accepted
Commit 6b73004668b3b71c9c38814b79fbb58c893ed434
Headers show
Series
  • [1/6] cve-update-db-native: don't hardcode the database name
Related show

Commit Message

Ross Burton Nov. 18, 2019, 4:46 p.m.
urllib handles adding proxy handlers if the proxies are set in the environment,
so call bb.utils.export_proxies() to do that and remove the manual setup.

Signed-off-by: Ross Burton <ross.burton@intel.com>

---
 .../recipes-core/meta/cve-update-db-native.bb | 31 +++----------------
 1 file changed, 5 insertions(+), 26 deletions(-)

-- 
2.20.1

-- 
_______________________________________________
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core

Comments

Mark Hatle Nov. 18, 2019, 4:51 p.m. | #1
On 11/18/19 10:46 AM, Ross Burton wrote:
> urllib handles adding proxy handlers if the proxies are set in the environment,

> so call bb.utils.export_proxies() to do that and remove the manual setup.

> 

> Signed-off-by: Ross Burton <ross.burton@intel.com>

> ---

>  .../recipes-core/meta/cve-update-db-native.bb | 31 +++----------------

>  1 file changed, 5 insertions(+), 26 deletions(-)

> 

> diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb

> index 08b18f064f0..db1d69a28e5 100644

> --- a/meta/recipes-core/meta/cve-update-db-native.bb

> +++ b/meta/recipes-core/meta/cve-update-db-native.bb

> @@ -21,10 +21,12 @@ python do_populate_cve_db() {

>      """

>      Update NVD database with json data feed

>      """

> -

> +    import bb.utils

>      import sqlite3, urllib, urllib.parse, shutil, gzip

>      from datetime import date

>  

> +    bb.utils.export_proxies(d)

> +

>      BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"

>      YEAR_START = 2002


Two comments, I know unrelated to this specific commit, but I noticed them while
looking...

The current NVD data is now in the '1.1' format.  I was lead to believe the 1.0
feeds would be stopped at some point.

Second, if we're successful with some of the SRTool components, we should be
able to export the data into NVD format.  So in that case, it would be nice to
be able to point the cve-update components to an alternative datasource.  (I do
assume the data format is the same.)

--Mark

> @@ -40,16 +42,6 @@ python do_populate_cve_db() {

>      except OSError:

>          pass

>  

> -    proxy = d.getVar("https_proxy")

> -    if proxy:

> -        # instantiate an opener but do not install it as the global

> -        # opener unless if we're really sure it's applicable for all

> -        # urllib requests

> -        proxy_handler = urllib.request.ProxyHandler({'https': proxy})

> -        proxy_opener = urllib.request.build_opener(proxy_handler)

> -    else:

> -        proxy_opener = None

> -

>      cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a')

>  

>      if not os.path.isdir(db_dir):

> @@ -67,15 +59,7 @@ python do_populate_cve_db() {

>          json_url = year_url + ".json.gz"

>  

>          # Retrieve meta last modified date

> -

> -        response = None

> -

> -        if proxy_opener:

> -            response = proxy_opener.open(meta_url)

> -        else:

> -            req = urllib.request.Request(meta_url)

> -            response = urllib.request.urlopen(req)

> -

> +        response = urllib.request.urlopen(meta_url)

>          if response:

>              for l in response.read().decode("utf-8").splitlines():

>                  key, value = l.split(":", 1)

> @@ -95,12 +79,7 @@ python do_populate_cve_db() {

>  

>              # Update db with current year json file

>              try:

> -                if proxy_opener:

> -                    response = proxy_opener.open(json_url)

> -                else:

> -                    req = urllib.request.Request(json_url)

> -                    response = urllib.request.urlopen(req)

> -

> +                response = urllib.request.urlopen(json_url)

>                  if response:

>                      update_db(c, gzip.decompress(response.read()).decode('utf-8'))

>                  c.execute("insert or replace into META values (?, ?)", [year, last_modified])

> 

-- 
_______________________________________________
Openembedded-core mailing list
Openembedded-core@lists.openembedded.org
http://lists.openembedded.org/mailman/listinfo/openembedded-core

Patch

diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb
index 08b18f064f0..db1d69a28e5 100644
--- a/meta/recipes-core/meta/cve-update-db-native.bb
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -21,10 +21,12 @@  python do_populate_cve_db() {
     """
     Update NVD database with json data feed
     """
-
+    import bb.utils
     import sqlite3, urllib, urllib.parse, shutil, gzip
     from datetime import date
 
+    bb.utils.export_proxies(d)
+
     BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
     YEAR_START = 2002
 
@@ -40,16 +42,6 @@  python do_populate_cve_db() {
     except OSError:
         pass
 
-    proxy = d.getVar("https_proxy")
-    if proxy:
-        # instantiate an opener but do not install it as the global
-        # opener unless if we're really sure it's applicable for all
-        # urllib requests
-        proxy_handler = urllib.request.ProxyHandler({'https': proxy})
-        proxy_opener = urllib.request.build_opener(proxy_handler)
-    else:
-        proxy_opener = None
-
     cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a')
 
     if not os.path.isdir(db_dir):
@@ -67,15 +59,7 @@  python do_populate_cve_db() {
         json_url = year_url + ".json.gz"
 
         # Retrieve meta last modified date
-
-        response = None
-
-        if proxy_opener:
-            response = proxy_opener.open(meta_url)
-        else:
-            req = urllib.request.Request(meta_url)
-            response = urllib.request.urlopen(req)
-
+        response = urllib.request.urlopen(meta_url)
         if response:
             for l in response.read().decode("utf-8").splitlines():
                 key, value = l.split(":", 1)
@@ -95,12 +79,7 @@  python do_populate_cve_db() {
 
             # Update db with current year json file
             try:
-                if proxy_opener:
-                    response = proxy_opener.open(json_url)
-                else:
-                    req = urllib.request.Request(json_url)
-                    response = urllib.request.urlopen(req)
-
+                response = urllib.request.urlopen(json_url)
                 if response:
                     update_db(c, gzip.decompress(response.read()).decode('utf-8'))
                 c.execute("insert or replace into META values (?, ?)", [year, last_modified])