Added pack-lint instructions to provide extended sanity checks.

Change-Id: Ibfe97b939e7b2e649c79fad68cc0d1dcc52a03f3

linter.py

@@ -0,0 +1,292 @@
+# -*- coding: utf-8 -*-
+
+import logging
+import lxml
+import os
+import os.path
+import re
+import requests
+
+from AdvancedHTMLParser import AdvancedHTMLParser
+from glob import iglob
+from urllib.parse import urlparse
+
+from cmsis.PackLint import PackLinter, VersionParser
+from cmsis.Pack import Pack, Api, SemanticVersion
+
+def create():
+  return CmsisPackLinter()
+
+class CmsisPackVersionParser(VersionParser):
+  def __init__(self, logger = None):
+    super().__init__(logger)
+    self._packs = {}
+
+  def _file_version_(self, file):
+    v = self._regex_(file, ".*@version\s+([vV])?([0-9]+[.][0-9]+([.][0-9]+)?).*", 2)
+    if not v:
+      v = self._regex_(file, ".*\$Revision:\s+([vV])?([0-9]+[.][0-9]+([.][0-9]+)?).*", 2)
+    return v
+    
+  def _revhistory_(self, file, skip = 0):
+    table = ""
+    dump = False
+    with open(file, 'r') as f:
+      for l in f:
+        if not dump and l.strip() == "<table class=\"cmtable\" summary=\"Revision History\">":
+          if skip > 0:
+            skip -= 1
+          else:
+            dump = True
+        if dump:
+          table += l.replace("<br>", "\\n")
+          if l.strip() == "</table>":
+            break
+    if table:
+      table = lxml.etree.fromstring(table)
+      m = re.match("[Vv]?(\d+.\d+(.\d+)?)", table[1][0].text)
+      if m:
+        return SemanticVersion(m.group(1))
+    else:
+      self._logger.info("Revision History table not found in "+file)
+    return None
+
+  def readme_md(self, file):
+    """Get the latest release version from README.md"""
+    return self._regex_(file, ".*repository contains the CMSIS Version ([0-9]+[.][0-9]+([.][0-9]+)?).*")
+
+  def _dxy(self, file):
+    """Get the PROJECT_NUMBER from a Doxygen configuration file."""
+    return self._regex_(file, "PROJECT_NUMBER\s*=\s*\"(Version\s+)?(\d+.\d+(.\d+)?)\"", 2)
+    
+  def _pdsc(self, file, component = None):
+    pack = None
+    if not file in self._packs:
+      pack = Pack(file, None)
+      self._packs[file] = pack
+    else:
+      pack = self._packs[file]
+    if component:
+      history = pack.history()
+      for r in sorted(history.keys(), reverse=True):
+        m = re.search(re.escape(component)+"(:)?\s+[Vv]?(\d+.\d+(.\d+)?)", history[r], re.MULTILINE)
+        if m:
+          return SemanticVersion(m.group(2))
+    else:
+      return pack.version()
+
+  def _h(self, file):
+    return self._file_version_(file)
+
+  def _c(self, file):
+    return self._file_version_(file)
+
+  def _s(self, file):
+    return self._file_version_(file)
+    
+  def overview_txt(self, file, skip = 0):
+    return self._revhistory_(file, skip)
+    
+  def introduction_txt(self, file, component = None):
+    if not component:
+      return None
+      
+    table = ""
+    dump = False
+    with open(file, 'r') as f:
+      for l in f:
+        if not dump and l.strip() == "<table class=\"cmtable\" summary=\"Revision History\">":
+          dump = True
+        if dump:
+          table += l.replace("<br>", "\\n")
+          if l.strip() == "</table>":
+            dump = False
+    table = lxml.etree.fromstring(table)
+    m = re.search(re.escape(component)+"\s+[Vv]?(\d+.\d+(.\d+)?)", table[1][1].text, re.MULTILINE)
+    if m:
+      return SemanticVersion(m.group(1))
+      
+    return None
+    
+  def dap_txt(self, file, skip = 0):
+    return self._revhistory_(file, skip)
+
+  def general_txt(self, file, skip = 0):
+    return self._revhistory_(file, skip)
+  
+  def history_txt(self, file, skip = 0):
+    return self._revhistory_(file, skip)
+
+  def _all_(self, file):
+    """Get the version or revision tag from an arbitrary file."""
+    version = self._regex_(file, ".*@version\s+([vV])?([0-9]+[.][0-9]+([.][0-9]+)?).*", 2)
+    if not version:
+      version = self._regex_(file, ".*\$Revision:\s+([vV])?([0-9]+[.][0-9]+([.][0-9]+)?).*", 2)
+    return version
+    
+class CmsisPackLinter(PackLinter):
+
+  def __init__(self, pdsc = "ARM.CMSIS.pdsc"):
+    super().__init__(pdsc)
+    self._versionParser = CmsisPackVersionParser(self._logger)
+    
+  def pack_version(self):
+    return self._pack.version()
+    
+  def cmsis_corem_component(self):
+    rte = { 'components' : set(), 'Dcore' : "Cortex-M3", 'Dvendor' : "", 'Dname' : "", 'Dtz' : "", 'Tcompiler' : "", 'Toptions' : "" }
+    comp = sorted(self._pack.component_by_name(rte, "CMSIS.CORE"), reverse=True)[0]
+    return SemanticVersion(comp.version())
+
+  def cmsis_corea_component(self):
+    rte = { 'components' : set(), 'Dcore' : "Cortex-A9", 'Dvendor' : "", 'Dname' : "", 'Dtz' : "", 'Tcompiler' : "", 'Toptions' : "" }
+    comp = sorted(self._pack.component_by_name(rte, "CMSIS.CORE"), reverse=True)[0]
+    return SemanticVersion(comp.version())
+
+  def cmsis_rtos2_api(self):
+    rte = { 'components' : set(), 'Dcore' : "", 'Dvendor' : "", 'Dname' : "", 'Dtz' : "", 'Tcompiler' : "", 'Toptions' : "" }
+    comp = sorted(self._pack.component_by_name(rte, "CMSIS.RTOS2"), reverse=True)[0]
+    return SemanticVersion(comp.version())
+
+  def cmsis_rtx5_component(self):
+    cs = self._pack.components_by_name("CMSIS.RTOS2.Keil RTX5*")
+    cvs = { (SemanticVersion(c.version()), SemanticVersion(c.apiversion())) for c in cs }
+    if len(cvs) == 1:
+      return cvs.pop()
+    elif len(cvs) > 1:
+      self.warning("Not all RTX5 components have same version information: %s", str([ (c.name(), c.version(), c.apiversion()) for c in cs ]))
+    return None, None
+
+  def check_general(self):
+    """CMSIS version"""
+    v = self.pack_version()
+    self.verify_version("README.md", v)
+    self.verify_version("CMSIS/DoxyGen/General/general.dxy", v)
+
+  def check_corem(self):
+    """CMSIS-Core(M) version"""
+    v = self.cmsis_corem_component()
+    self.verify_version("CMSIS/DoxyGen/Core/core.dxy", v)
+    self.verify_version("CMSIS/DoxyGen/Core/src/Overview.txt", v)
+    self.verify_version("CMSIS/DoxyGen/General/src/introduction.txt", v, component="CMSIS-Core (Cortex-M)")
+    self.verify_version(self._pack.location(), v, component="CMSIS-Core(M)")
+
+  def check_corea(self):
+    """CMSIS-Core(A) version"""
+    v = self.cmsis_corea_component()
+    self.verify_version("CMSIS/DoxyGen/Core_A/core_A.dxy", v)
+    self.verify_version("CMSIS/DoxyGen/Core_A/src/Overview.txt", v)
+    self.verify_version("CMSIS/DoxyGen/General/src/introduction.txt", v, component="CMSIS-Core (Cortex-A)")
+    self.verify_version(self._pack.location(), v, component="CMSIS-Core(A)")
+
+  def check_dap(self):
+    """CMSIS-DAP version"""
+    v = self._versionParser.get_version("CMSIS/DoxyGen/DAP/dap.dxy")
+    self.verify_version("CMSIS/DoxyGen/DAP/src/dap.txt", v)
+    self.verify_version("CMSIS/DoxyGen/General/src/introduction.txt", v, component="CMSIS-DAP")
+    self.verify_version(self._pack.location(), v, component="CMSIS-DAP")
+
+  def check_driver(self):
+    """CMSIS-Driver version"""
+    v = self._versionParser.get_version("CMSIS/DoxyGen/Driver/Driver.dxy")
+    self.verify_version("CMSIS/DoxyGen/Driver/src/General.txt", v)
+    self.verify_version("CMSIS/DoxyGen/General/src/introduction.txt", v, component="CMSIS-Driver")
+    self.verify_version(self._pack.location(), v, component="CMSIS-Driver")
+
+  def check_dsp(self):
+    """CMSIS-DSP version"""
+    v = self._versionParser.get_version("CMSIS/DoxyGen/DSP/dsp.dxy")
+    self.verify_version("CMSIS/DoxyGen/DSP/src/history.txt", v)
+    self.verify_version("CMSIS/DoxyGen/General/src/introduction.txt", v, component="CMSIS-DSP")
+    self.verify_version(self._pack.location(), v, component="CMSIS-DSP")
+
+  def check_nn(self):
+    """CMSIS-NN version"""
+    v = self._versionParser.get_version("CMSIS/DoxyGen/NN/nn.dxy")
+    self.verify_version("CMSIS/DoxyGen/NN/src/history.txt", v)
+    self.verify_version("CMSIS/DoxyGen/General/src/introduction.txt", v, component="CMSIS-NN")
+    self.verify_version(self._pack.location(), v, component="CMSIS-NN")
+
+  def check_pack(self):
+    """CMSIS-Pack version"""
+    v = self._versionParser.get_version("CMSIS/DoxyGen/Pack/Pack.dxy")
+    self.verify_version("CMSIS/DoxyGen/Pack/src/General.txt", v)
+    self.verify_version("CMSIS/DoxyGen/General/src/introduction.txt", v, component="CMSIS-Pack")
+    self.verify_version(self._pack.location(), v, component="CMSIS-Pack")
+
+  def check_rtos2(self):
+    """CMSIS-RTOS2 version"""
+    api = self.cmsis_rtos2_api()
+    v, a = self.cmsis_rtx5_component()
+    self.verify_version("CMSIS/DoxyGen/RTOS2/rtos.dxy", api)
+    self.verify_version("CMSIS/DoxyGen/RTOS2/src/history.txt", api, skip=0)
+    self.verify_version("CMSIS/DoxyGen/General/src/introduction.txt", api, component="CMSIS-RTOS")
+    # self.verify_version(self._pack.location(), v, component="CMSIS-RTOS2")
+    if a and not api.match(a):
+      self.warning("RTX5 API version (%s) does not match RTOS2 API version (%s)!", a, api)
+    self.verify_version("CMSIS/DoxyGen/RTOS2/src/history.txt", v, skip=1)
+    
+  def check_files(self):
+    """Files referenced by pack description"""
+    # Check schema of pack description
+    self.verify_schema(self._pack.location(), "CMSIS/Utilities/PACK.xsd")
+        
+    # Check schema of SVD files
+    svdfiles = { d.svdfile() for d in self._pack.devices() if d.svdfile() }
+    for svd in svdfiles:
+      if os.path.exists(svd):
+        self.verify_schema(svd, "CMSIS/Utilities/CMSIS-SVD.xsd")
+      else:
+        self.warning("SVD File does not exist: %s!", svd)
+
+    # Check component file version
+    for c in self._pack.components():
+      cv = c.version()
+      for f in c.files():
+        hv = f.version()
+        if c is Api:
+          if f.isHeader():
+            if not hv:
+              self.verify_version(f.location(), cv)
+        if hv:
+          self.verify_version(f.location(), hv)
+  
+  def check_doc(self, pattern="./gen_pack/CMSIS/Documentation/**/*.html"):
+    """Documentation"""
+    self.debug("Using pattern '%s'", pattern)
+    for html in iglob(pattern, recursive=True):
+      self.info("%s: Checking links ...", html)
+      parser = AdvancedHTMLParser()
+      parser.parseFile(html)
+      links = parser.getElementsByTagName("a")
+      for l in links:
+        href = l.getAttribute("href")
+        if href:
+          href = urlparse(href)
+          if href.scheme in ["http", "https", "ftp", "ftps" ]:
+            try:
+              self.info("%s: Checking link to %s...", html, href.geturl())
+              r = requests.head(href.geturl(), headers={'user-agent' : "packlint/1.0"}, timeout=10)
+              r.raise_for_status()
+            except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
+              exc_info = None
+              if self.loglevel() == logging.DEBUG:
+                exc_info = e
+              self.warning("%s: Broken web-link to %s!", html, href.geturl(), exc_info=exc_info)
+            except requests.exceptions.Timeout as e:
+              exc_info = None
+              if self.loglevel() == logging.DEBUG:
+                exc_info = e
+              self.warning("%s: Timeout following web-link to %s.", html, href.geturl(), exc_info=exc_info)
+          elif href.scheme == "javascript":
+            pass
+          elif not os.path.isabs(href.path):
+            target = os.path.normpath(os.path.join(os.path.dirname(html), href.path))
+            if not os.path.exists(target):
+              self.warning("%s: Broken relative-link to %s!", html, href.path)
+          else:
+            self.warning("%s: Broken relative-link to %s!", html, href.path)
+  
+  def check_schema(self):
+    """XML Schema"""
+    pass