Merge pull request #24 from Th0ught09/staging

pre-commit installed and formatted files

.gitignore

@@ -145,3 +145,9 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
+
+# IDEs
+.idea/
+
+# Tag files
+tags

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: mixed-line-ending

README.md

@@ -84,9 +84,13 @@ Linux-based system.
 
 - [Docker](https://docs.docker.com/engine/install/)
 - python3 (should work with any version of Python 3, known to work with at least 3.10.5)
-- [python-git](https://pypi.org/project/python-git/) (`pip3 install gitpython`, known to work with at least 3.1.27)
 - a recent-enough Linux kernel to support seccomp and ptrace (i.e., if your Linux kernel doesn't support them, you really seriously should update your setup :innocent:)
 
+Python dependencies which may be installed using `pip install -r requirements.txt`
+
+- [python-git](https://pypi.org/project/python-git/)
+- [pre-commit](https://pre-commit.com/)
+
 Once these dependencies have been installed, the setup is very simple: `make
 all`
 
@@ -376,7 +380,7 @@ main wrapper, and may not be stable.
 
 ### Integration with debhelper
 
-We provide a script that integrates Loupe with [`debhelper`](https://man7.org/linux/man-pages/man7/debhelper.7.html). 
+We provide a script that integrates Loupe with [`debhelper`](https://man7.org/linux/man-pages/man7/debhelper.7.html).
 Our script automatically downloads the debian sources of a package, builds it and then runs the test suite with loupe (if supported by the package).
 We use `dh_test_auto` to run the testsuite. `dh_test_auto` returns 0 if the test suite executed successfully.
 
@@ -641,6 +645,12 @@ Here are a few ideas of contributions to get started:
 - Add support to convert Loupe databases to SQLite.
 - Fix [open bug reports](https://github.com/unikraft/loupe/issues).
 
+It is recommended to make use of the [pre-commit](https://pre-commit.com/) to ensure consistent formatting between files. You should have already installed pre-commit from within the `requirements.txt`, after which run:
+```bash
+pre-commit install
+```
+Which will ensure that whenever you make a commit, all files will be automatically formatted.
+
 ## 10. Disclaimer
 
 This artifact is the first release of a research proof-of-concept for Loupe.

misc-artifacts/implementation-timeline/print-timeline.py

@@ -221,4 +221,3 @@ def time_to_reach(os_data, syscall_num):
             print("----------- " + os + " below -----------")
             scores[os] = get_scores_in_time(data[os])
             print_timeline(data[os], scores[os])
-

misc-artifacts/implementation-timeline/top-syscalls.py

@@ -106,11 +106,3 @@ def get_top_all(data, top):
         topdata = get_top_all(data, top)
         print("top " + str(top) + ": " + str(topdata) + " (total " + \
                 str(len(topdata)) + ")")
-
-
-
-
-
-
-
-

misc-artifacts/os-compatibility-score/count-usage.sh

@@ -105,4 +105,3 @@ for p in "all" "suite" "bench"; do
 done
 
 echo "Done!"
-

misc-artifacts/os-compatibility-score/google-csvs/graphene.csv

@@ -333,4 +333,4 @@ Syscall N#,Syscall name,Status,Remark,
 331,pkey_free,,,
 332,statx,,,
 333,io_pgetevents,,,
-334,rseq,,,
+334,rseq,,,

misc-artifacts/os-compatibility-score/google-csvs/gvisor.csv

@@ -333,4 +333,4 @@ Syscall N#,Syscall name,Status,
 331,pkey_free,not implemented,
 332,statx,implemented,
 333,io_pgetevents,not implemented,
-334,rseq,implemented,partially supported
+334,rseq,implemented,partially supported

misc-artifacts/os-compatibility-score/google-csvs/hermitux.csv

@@ -333,4 +333,4 @@ Syscall N#,Syscall name,Status
 331,pkey_free,not implemented
 332,statx,not implemented
 333,io_pgetevents,not implemented
-334,rseq,not implemented
+334,rseq,not implemented

misc-artifacts/os-compatibility-score/google-csvs/osv.csv

@@ -333,4 +333,4 @@ Syscall N#,Syscall name,Status,,Implemented,Stubbed,fs API,Notes
 331,pkey_free,Not implemented,,,,,
 332,statx,Not implemented,,,,,
 333,io_pgetevents,Not implemented,,,,,
-334,rseq,Not implemented,,,,,
+334,rseq,Not implemented,,,,,

misc-artifacts/os-compatibility-score/google-csvs/unikraft.csv

@@ -333,4 +333,4 @@ Syscall N#,Syscall name,Status,,Implemented,stubbed,Incomplete
 331,pkey_free,Not implemented,,,,
 332,statx,Not implemented,,,,
 333,io_pgetevents,Not implemented,,,,
-334,rseq,Not implemented,,,,
+334,rseq,Not implemented,,,,

misc-artifacts/os-compatibility-score/google-csvs/zephyr.csv

@@ -333,4 +333,4 @@ Syscall N#,Syscall name,Status,Remark,
 331,pkey_free,,,
 332,statx,,,
 333,io_pgetevents,,,
-334,rseq,,,
+334,rseq,,,

requirements.txt

@@ -0,0 +1,2 @@
+pre-commit==4.0.1
+gitpython==3.1.27

src/static-source-analyser/check_syscall.py

@@ -44,14 +44,14 @@ def addSyscalls(syscallName, covFolder, used):
     else:
         covFolder.notCovSyscalls.add(syscallName)
         covFolder.manualSetSyscalls.add(syscallName)
-            
+
 def readCsvManual(covFolder):
-    
+
     if COVERAGE_SUITE in covFolder.htmlFolder:
         colIndex = 2
     else:
         colIndex = 1
-    
+
     try:
         csvReader = csv.reader(open(covFolder.csvFile, 'r'), delimiter=',')
         for row in csvReader:
@@ -61,4 +61,3 @@ def readCsvManual(covFolder):
     except:
         print("[Error]: cannot read the CSV file: " + covFolder.csvFile)
         pass
-

src/static-source-analyser/output_html.py

@@ -42,7 +42,7 @@ def saveResultsHtml(output, covFolder):
     isExist = os.path.exists(htmlPath)
     if not isExist:
         os.makedirs(htmlPath)
-        
+
     for name, htmlFile in covFolder.mapHtmlFile.items():
         if len(htmlFile.linesNotCov) > 0:
             with open(os.path.join(htmlPath, htmlFile.name), "w") as f:
@@ -85,7 +85,7 @@ def checkIncludeFile(gObj, key, appName):
     return info
 
 def saveAggregateHtmlUnique(gObj, maxDisplay, outAggregatedUnique, covFolder, resultsFolder):
-    
+
     strBuilder = StringIO()
     i = 0
     total = len(covFolder.allSyscalls)
@@ -109,22 +109,22 @@ def saveAggregateHtmlUnique(gObj, maxDisplay, outAggregatedUnique, covFolder, re
             strBuilder.write("\t<tr><td><a href=" + os.path.join("pdf_files", s + ".pdf") + ">" + s + "</a></td></tr>\n")
             j += 1
     strBuilder.write("</tbody></table></div>\n")
-    
+
     #for s in covFolder.allSyscalls:
     #    if s not in covFolder.covSyscalls and s not in covFolder.notCovSyscalls:
     #        print(s)
 
     total = i+j
     strBuilder.write("\n<div><hr><p>Syscalls coverage: {:.2f}".format((i/total) * 100)+ "%</p><hr></div>")
     print("Syscalls coverage: {}/{} = {:.2f}".format(i, total, (i/total) * 100))
-    
+
     with open(os.path.join(resultsFolder, "small.html"), "w") as file:
         file.write("<hr><div><h3>Static syscalls only (not compiled in the final binary)</h3><table><tbody>")
         for s in covFolder.allSyscalls:
             if s not in covFolder.notCovSyscalls and s not in covFolder.covSyscalls:
                 file.write("<tr><td><a href=" + os.path.join("pdf_files", s + ".pdf") + ">" + s + "</a></td></tr>")
         file.write("</tbody></table></div>")
-    
+
     strBuilder.write("\n<br><div><h3>functions not covered</h3><div><p><i>st</i>: static - <i>mc</i>: macro - <i>unk</i>: unknown</p></div><table><tbody>")
     i = 0
     for key, values in covFolder.notCovFct.items():
@@ -150,6 +150,6 @@ def saveAggregateHtmlUnique(gObj, maxDisplay, outAggregatedUnique, covFolder, re
         i += 1
 
     strBuilder.write("</tbody></table></div></body></html>")
-    
+
     with open(outAggregatedUnique, "w") as f:
-        f.write(strBuilder.getvalue())
+        f.write(strBuilder.getvalue())

src/static-source-analyser/parser_clang.py

@@ -175,7 +175,7 @@ def find_funcs_and_calls(tu):
             funcs.append(c)
             # filter name to take only the name if necessary
             funcName = filter_func_name(c.displayname)
-            
+
             #increment counter
             if funcName not in global_funcs:
                 global_funcs[funcName].append(1)
@@ -223,7 +223,7 @@ def compare_syscalls(syscalls):
     define_syscalls = defaultdict(list)
 
     for key, value in global_calls.items():
-        
+
         if key in syscalls or key in alias_syscalls_list:
             print_verbose(key,verbose)
             called_syscalls[key] = value
@@ -245,7 +245,7 @@ def compare_syscalls(syscalls):
     return (called_syscalls, define_syscalls)
 
 def process_source_code(folder, v):
-    global verbose 
+    global verbose
     verbose = v
     includePaths = get_include_paths(folder)
     check_input_path(folder, includePaths)
@@ -273,21 +273,21 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--folder','-f', help='Path to the folder (source files) of the application to analyse', required=True)
     parser.add_argument('--output', '-o', help='Path to the output resulting json file')
-    parser.add_argument('--verbose', '-v', type=str2bool, 
+    parser.add_argument('--verbose', '-v', type=str2bool,
                         nargs='?', const=True, default=False,
                         help='Verbose mode')
-    parser.add_argument('--display', '-d', type=str2bool, 
+    parser.add_argument('--display', '-d', type=str2bool,
                         nargs='?', const=True, default=False,
                         help='Display all syscalls')
     args = parser.parse_args()
-    
+
     verbose = args.verbose
     output_dict = process_source_code(args.folder, args.verbose)
     if args.output is None:
         output_file = sys.stdout
     else:
         output_file = open(args.output, "w")
-        
+
     if args.display:
         for s in output_dict['all_system_calls']:
             print(s)
@@ -303,4 +303,4 @@ def main():
         if not os.path.isfile(filename):
             print_err("Cannot find {}. Please install clang shared library. See README.md for more information.".format(filename))
 
-    main()
+    main()

src/static-source-analyser/process_call.py

@@ -379,7 +379,7 @@ def buildAll(gObj, args):
     full_call_graph(gObj.functions, exclude=getExcludeRegex(args), no_externs=args.no_externs)
 
 def buildCaller(gObj, covFolder, caller_lst, args, isCovered=False):
-    
+
     for caller in caller_lst:
         if caller not in gObj.functions:
             print_warn("Can't find caller \"{}\" in RTL data!".format(caller))
@@ -454,4 +454,4 @@ def buildCallee(gObj, covFolder, callee_lst, args):
             except subprocess.TimeoutExpired:
                 print_warn("60sec timer expired for " + dotFile)
             except FileNotFoundError:
-                print_err("dot command not found! Please install graphviz.")
+                print_err("dot command not found! Please install graphviz.")

src/static-source-analyser/source_analyser.py

@@ -73,7 +73,7 @@ def search_function(line, cv):
             # It is only a keyword so return
             print_verbose("[" + cv +"] Ignore: [" + matches[0].replace(" ", "") + "]\n-------", verbose)
             return None
-        
+
         print_verbose("[" + cv +"] Take:   [" + matches[0].replace(" ", "") + "]\n-------", verbose)
 
         htmlLine = HtmlLine(line)
@@ -99,7 +99,7 @@ def getHtmlLines(filename, covFolder):
 
         for span in htmlContent.find_all("span", {"class": LINECOV}):
             htmlLine = search_function(span.get_text(), LINECOV)
-            
+
             if htmlLine:
                 htmlFile.linesCov.append(htmlLine)
                 for fct in htmlLine.fctList:
@@ -109,7 +109,7 @@ def getHtmlLines(filename, covFolder):
                     try:
                         covFolder.covFct[fct].add(filename + "#" + str(span.parent['name']))
                     except:
-                        covFolder.covFct[fct].add(filename)             
+                        covFolder.covFct[fct].add(filename)
 
         for span in htmlContent.find_all("span", {"class": LINENOCOV}):
             htmlLine = search_function(span.get_text(), LINENOCOV)
@@ -124,7 +124,7 @@ def getHtmlLines(filename, covFolder):
                         covFolder.notCovFct[fct].add(filename + "#" + str(span.parent['name']))
                     except:
                         covFolder.notCovFct[fct].add(filename)
-    
+
     covFolder.mapHtmlFile[filename] = htmlFile
 
 def iterateHtmlFolder(covFolder):
@@ -142,7 +142,7 @@ def iterateExtandFolder(gObj):
 
     pathlist = Path(gObj.expandFolder).glob('**/*.expand')
     for path in pathlist:
-        
+
         str_path = str(path)
         print_verbose("Gathering info of: " + str(str_path), verbose)
         if not os.path.isfile(str_path) or not os.access(str_path, os.R_OK):
@@ -218,7 +218,7 @@ def main():
     # Use clang parser to analyse source code
     print("[INFO] Perfoming clang analysis... (this may take some times...)")
     output_dict = process_source_code(args.folder, args.verbose)
-    
+
     # Build the graph
     if expandFolder != None:
         gObj = GraphObject(expandFolder)
@@ -232,7 +232,7 @@ def main():
             # Add pdf and dot folders to gCov
             gObj.outDotFolder = os.path.join(resultsFolder, "dot_files")
             gObj.outPdfFolder = os.path.join(resultsFolder, "pdf_files")
-        
+
             # Manual inspection of data (sanitize)
             if os.path.exists(os.path.join(args.folder, appName + ".csv")):
                 readCsvManual(covFolder)
@@ -259,7 +259,7 @@ def main():
         for s in output_dict['all_system_calls']:
             print(s)
 
-    # Write syscalls in a CSV file with the following format (syscall number, covered {Y|N})        
+    # Write syscalls in a CSV file with the following format (syscall number, covered {Y|N})
     if args.csv:
         with open(os.path.join(args.folder, "syscalls_" + appName + ".csv"), "w") as f:
             for k,v in syscall_list.items():
@@ -278,4 +278,4 @@ def main():
                 print(obj)
 
 if __name__== "__main__":
-    main()
+    main()

src/static-source-analyser/syscalls_list.py

@@ -396,4 +396,4 @@
     "ftruncate64" : "ftruncate",
     "setrlimit64" : "setrlimit",
     "sendfile64" : "sendfile",
-}
+}

src/static-source-analyser/utility.py

@@ -66,4 +66,4 @@ def print_buf(str_graph, buf, text):
 def createFolder(filename):
     isExist = os.path.exists(filename)
     if not isExist:
-        os.makedirs(filename)
+        os.makedirs(filename)