Merge pull request #1544 from hathach/ci-parallel-build

Ci parallel build

.github/workflows/build_aarch64.yml

@@ -55,6 +55,9 @@ jobs:
     - name: Set Toolchain Path
       run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin`
 
+    - name: Get Dependencies
+      run: python3 tools/get_dependencies.py ${{ matrix.family }}
+
     - name: Build
       run: python3 tools/build_family.py ${{ matrix.family }}
 

.github/workflows/build_arm.yml

@@ -67,6 +67,11 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v3
 
+    - name: Install ARM GCC
+      uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+        release: '11.2-2022.02'
+
     - name: Checkout TinyUSB
       uses: actions/checkout@v3
 
@@ -84,12 +89,9 @@ jobs:
       run: |
         git clone --depth 1 -b develop https://github.com/raspberrypi/pico-sdk ~/pico-sdk
         echo >> $GITHUB_ENV PICO_SDK_PATH=~/pico-sdk
-        git submodule update --init hw/mcu/raspberry_pi/Pico-PIO-USB
 
-    - name: Install ARM GCC
-      uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '11.2-2022.02'
+    - name: Get Dependencies
+      run: python3 tools/get_dependencies.py ${{ matrix.family }}
 
     - name: Build
       run: python3 tools/build_family.py ${{ matrix.family }}
@@ -122,16 +124,16 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v3
 
+    - name: Install ARM GCC
+      uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+        release: '11.2-2022.02'
+
     - name: Checkout TinyUSB
       uses: actions/checkout@v3
 
     - name: Checkout common submodules in lib
       run: git submodule update --init lib/FreeRTOS-Kernel lib/lwip
 
-    - name: Install ARM GCC
-      uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '11.2-2022.02'
-
     - name: Build
       run: python3 tools/build_board.py ${{ matrix.example }}

.github/workflows/build_msp430.yml

@@ -52,6 +52,9 @@ jobs:
     - name: Set Toolchain Path
       run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin`
 
+    - name: Get Dependencies
+      run: python3 tools/get_dependencies.py ${{ matrix.family }}
+
     - name: Build
       run: python3 tools/build_family.py ${{ matrix.family }}
 

.github/workflows/build_renesas.yml

@@ -53,6 +53,9 @@ jobs:
     - name: Set Toolchain Path
       run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin`
 
+    - name: Get Dependencies
+      run: python3 tools/get_dependencies.py ${{ matrix.family }}
+
     - name: Build
       run: python3 tools/build_family.py ${{ matrix.family }}
 

.github/workflows/build_riscv.yml

@@ -53,6 +53,9 @@ jobs:
     - name: Set Toolchain Path
       run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin`
 
+    - name: Get Dependencies
+      run: python3 tools/get_dependencies.py ${{ matrix.family }}
+
     - name: Build
       run: python3 tools/build_family.py ${{ matrix.family }}
 

docs/reference/getting_started.rst

@@ -50,7 +50,12 @@ Some TinyUSB examples also requires external submodule libraries in ``/lib`` suc
 
    $ git submodule update --init lib
 
-In addition, MCU driver submodule is also needed to provide low-level MCU peripheral's driver. Luckily, it will be fetched if needed when you run the ``make`` to build your board.
+In addition, MCU driver submodule is also needed to provide low-level MCU peripheral's driver. To download these depencies for your board, run the ``get-dpes`` as follow.
+
+.. code-block::
+
+   $ make BOARD=feather_nrf52840_express get-deps
+
 
 Some modules will also require a module-specific SDK (e.g. RP2040) or binary (e.g. Sony Spresense) to build examples.
 

examples/make.mk

@@ -45,11 +45,6 @@ else
   SRC_C += $(subst $(TOP)/,,$(wildcard $(TOP)/$(FAMILY_PATH)/*.c))
 endif
 
-# Fetch submodules depended by family
-fetch_submodule_if_empty = $(if $(wildcard $(TOP)/$1/*),,$(info $(shell git -C $(TOP) submodule update --init $1)))
-ifdef DEPS_SUBMODULES
-  $(foreach s,$(DEPS_SUBMODULES),$(call fetch_submodule_if_empty,$(s)))
-endif
 
 #-------------- Cross Compiler  ------------
 # Can be set by board, default to ARM GCC

examples/rules.mk

@@ -5,6 +5,7 @@
 # Set all as default goal
 .DEFAULT_GOAL := all
 
+# ---------------- GNU Make Start -----------------------
 # ESP32-Sx and RP2040 has its own CMake build system
 ifeq (,$(findstring $(FAMILY),esp32s2 esp32s3 rp2040))
 
@@ -141,7 +142,23 @@ $(BUILD)/obj/%_asm.o: %.S
 	@echo AS $(notdir $@)
 	@$(CC) -x assembler-with-cpp $(ASFLAGS) -c -o $@ $<
 
-endif # GNU Make
+endif
+
+.PHONY: clean
+clean:
+ifeq ($(CMDEXE),1)
+	rd /S /Q $(subst /,\,$(BUILD))
+else
+	$(RM) -rf $(BUILD)
+endif
+# ---------------- GNU Make End -----------------------
+
+# get depenecies
+.PHONY: get-deps
+get-deps:
+  ifdef DEPS_SUBMODULES
+	git -C $(TOP) submodule update --init $(DEPS_SUBMODULES)
+  endif
 
 size: $(BUILD)/$(PROJECT).elf
 	-@echo ''
@@ -152,14 +169,6 @@ size: $(BUILD)/$(PROJECT).elf
 linkermap: $(BUILD)/$(PROJECT).elf
 	@linkermap -v $<.map
 
-.PHONY: clean
-clean:
-ifeq ($(CMDEXE),1)
-	rd /S /Q $(subst /,\,$(BUILD))
-else
-	$(RM) -rf $(BUILD)
-endif
-
 # ---------------------------------------
 # Flash Targets
 # ---------------------------------------

hw/bsp/rp2040/family.mk

@@ -1,6 +1,8 @@
 JLINK_DEVICE = rp2040_m0_0
 PYOCD_TARGET = rp2040
 
+DEPS_SUBMODULES += hw/mcu/raspberry_pi/Pico-PIO-USB
+
 ifeq ($(DEBUG), 1)
 CMAKE_DEFSYM += -DCMAKE_BUILD_TYPE=Debug
 endif

tools/build_board.py

@@ -1,8 +1,8 @@
 import os
-import glob
 import sys
-import subprocess
 import time
+import subprocess
+from multiprocessing import Pool
 
 import build_utils
 
@@ -10,90 +10,60 @@ SUCCEEDED = "\033[32msucceeded\033[0m"
 FAILED = "\033[31mfailed\033[0m"
 SKIPPED = "\033[33mskipped\033[0m"
 
-success_count = 0
-fail_count = 0
-skip_count = 0
-exit_status = 0
-
-total_time = time.monotonic()
-
-build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
 build_separator = '-' * 106
 
+
 def filter_with_input(mylist):
     if len(sys.argv) > 1:
         input_args = list(set(mylist).intersection(sys.argv))
         if len(input_args) > 0:
             mylist[:] = input_args
 
-# If examples are not specified in arguments, build all
-all_examples = []
-for dir1 in os.scandir("examples"):
-    if dir1.is_dir():
-        for entry in os.scandir(dir1.path):
-            if entry.is_dir():
-                all_examples.append(dir1.name + '/' + entry.name)
-filter_with_input(all_examples)
-all_examples.sort()
 
-# If boards are not specified in arguments, build all
-all_boards = []
-for entry in os.scandir("hw/bsp"):
-    if entry.is_dir() and os.path.exists(entry.path + "/board.mk"):
-        all_boards.append(entry.name)
-filter_with_input(all_boards)
-all_boards.sort()
+if __name__ == '__main__':
+    # If examples are not specified in arguments, build all
+    all_examples = []
+    for dir1 in os.scandir("examples"):
+        if dir1.is_dir():
+            for entry in os.scandir(dir1.path):
+                if entry.is_dir():
+                    all_examples.append(dir1.name + '/' + entry.name)
+    filter_with_input(all_examples)
+    all_examples.sort()
 
-def build_board(example, board):
-    global success_count, fail_count, skip_count, exit_status
-    start_time = time.monotonic()
-    flash_size = "-"
-    sram_size = "-"
+    # If boards are not specified in arguments, build all
+    all_boards = []
+    for entry in os.scandir("hw/bsp"):
+        if entry.is_dir() and os.path.exists(entry.path + "/board.mk"):
+            all_boards.append(entry.name)
+    filter_with_input(all_boards)
+    all_boards.sort()
 
-    # Check if board is skipped
-    if build_utils.skip_example(example, board):
-        success = SKIPPED
-        skip_count += 1
-        print(build_format.format(example, board, success, '-', flash_size, sram_size))
-    else:
-        subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    # Get dependencies
+    for b in all_boards:
+        subprocess.run("make -C examples/device/board_test BOARD={} get-deps".format(b), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
-        if build_result.returncode == 0:
-            success = SUCCEEDED
-            success_count += 1
-            (flash_size, sram_size) = build_size(example, board)
-        else:
-            exit_status = build_result.returncode
-            success = FAILED
-            fail_count += 1
-
-        build_duration = time.monotonic() - start_time
-        print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
-
-        if build_result.returncode != 0:
-            print(build_result.stdout.decode("utf-8"))
-
-def build_size(example, board):
-    #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board)
-    elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
-    size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
-    size_list = size_output.split('\n')[1].split('\t')
-    flash_size = int(size_list[0])
-    sram_size = int(size_list[1]) + int(size_list[2])
-    return (flash_size, sram_size)
-
-print(build_separator)
-print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
-
-for example in all_examples:
     print(build_separator)
-    for board in all_boards:
-        build_board(example, board)
+    print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+    total_time = time.monotonic()
 
-total_time = time.monotonic() - total_time
-print(build_separator)
-print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time))
-print(build_separator)
+    # succeeded, failed, skipped
+    total_result = [0, 0, 0]
+    for example in all_examples:
+        print(build_separator)
+        with Pool(processes=os.cpu_count()) as pool:
+            pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
+            result = pool.starmap(build_utils.build_example, pool_args)
+            # sum all element of same index (column sum)
+            result = list(map(sum, list(zip(*result))))
+            
+            # add to total result
+            total_result = list(map(lambda x, y: x + y, total_result, result))
 
-sys.exit(exit_status)
+    total_time = time.monotonic() - total_time
+    print(build_separator)
+    print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
+                                                                       FAILED, total_result[2], SKIPPED, total_time))
+    print(build_separator)
+
+    sys.exit(total_result[1])

tools/build_family.py

@@ -1,8 +1,7 @@
 import os
-import glob
 import sys
-import subprocess
 import time
+from multiprocessing import Pool
 
 import build_utils
 
@@ -10,40 +9,15 @@ SUCCEEDED = "\033[32msucceeded\033[0m"
 FAILED = "\033[31mfailed\033[0m"
 SKIPPED = "\033[33mskipped\033[0m"
 
-success_count = 0
-fail_count = 0
-skip_count = 0
-exit_status = 0
-
-total_time = time.monotonic()
-
-build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
 build_separator = '-' * 106
 
+
 def filter_with_input(mylist):
     if len(sys.argv) > 1:
         input_args = list(set(mylist).intersection(sys.argv))
         if len(input_args) > 0:
             mylist[:] = input_args
 
-# If examples are not specified in arguments, build all
-all_examples = []
-for dir1 in os.scandir("examples"):
-    if dir1.is_dir():
-        for entry in os.scandir(dir1.path):
-            if entry.is_dir():
-                all_examples.append(dir1.name + '/' + entry.name)
-filter_with_input(all_examples)
-all_examples.sort()
-
-# If family are not specified in arguments, build all
-all_families = []
-for entry in os.scandir("hw/bsp"):
-    if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"):
-        all_families.append(entry.name)
-            
-filter_with_input(all_families)
-all_families.sort()
 
 def build_family(example, family):
     all_boards = []
@@ -52,61 +26,49 @@ def build_family(example, family):
             all_boards.append(entry.name)
     filter_with_input(all_boards)
     all_boards.sort()
-    
-    for board in all_boards:
-        build_board(example, board)
-    
-def build_board(example, board):
-    global success_count, fail_count, skip_count, exit_status
-    start_time = time.monotonic()
-    flash_size = "-"
-    sram_size = "-"
 
-    # Check if board is skipped
-    if build_utils.skip_example(example, board):
-        success = SKIPPED
-        skip_count += 1
-        print(build_format.format(example, board, success, '-', flash_size, sram_size))
-    else:   
-        #subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    with Pool(processes=os.cpu_count()) as pool:
+        pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
+        result = pool.starmap(build_utils.build_example, pool_args)
+        # sum all element of same index (column sum)
+        return list(map(sum, list(zip(*result))))
 
-        if build_result.returncode == 0:
-            success = SUCCEEDED
-            success_count += 1
-            (flash_size, sram_size) = build_size(example, board)
-            subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        else:
-            exit_status = build_result.returncode
-            success = FAILED
-            fail_count += 1
 
-        build_duration = time.monotonic() - start_time
-        print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
+if __name__ == '__main__':
+    # If examples are not specified in arguments, build all
+    all_examples = []
+    for dir1 in os.scandir("examples"):
+        if dir1.is_dir():
+            for entry in os.scandir(dir1.path):
+                if entry.is_dir():
+                    all_examples.append(dir1.name + '/' + entry.name)
+    filter_with_input(all_examples)
+    all_examples.sort()
 
-        if build_result.returncode != 0:
-            print(build_result.stdout.decode("utf-8"))
+    # If family are not specified in arguments, build all
+    all_families = []
+    for entry in os.scandir("hw/bsp"):
+        if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"):
+            all_families.append(entry.name)
+    filter_with_input(all_families)
+    all_families.sort()
 
-def build_size(example, board):
-    #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board)
-    elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
-    size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
-    size_list = size_output.split('\n')[1].split('\t')
-    flash_size = int(size_list[0])
-    sram_size = int(size_list[1]) + int(size_list[2])
-    return (flash_size, sram_size)
-
-print(build_separator)
-print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
-
-for example in all_examples:
     print(build_separator)
-    for family in all_families:
-        build_family(example, family)
+    print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+    total_time = time.monotonic()
 
-total_time = time.monotonic() - total_time
-print(build_separator)
-print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time))
-print(build_separator)
+    # succeeded, failed, skipped
+    total_result = [0, 0, 0]
+    for example in all_examples:
+        print(build_separator)
+        for family in all_families:
+            fret = build_family(example, family)
+            total_result = list(map(lambda x, y: x + y, total_result, fret))
 
-sys.exit(exit_status)
+    total_time = time.monotonic() - total_time
+    print(build_separator)
+    print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
+                                                                       FAILED, total_result[2], SKIPPED, total_time))
+    print(build_separator)
+
+    sys.exit(total_result[1])

tools/build_utils.py

@@ -1,9 +1,18 @@
+import subprocess
 import pathlib
+import time
+
+build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
+
+SUCCEEDED = "\033[32msucceeded\033[0m"
+FAILED = "\033[31mfailed\033[0m"
+SKIPPED = "\033[33mskipped\033[0m"
+
 
 def skip_example(example, board):
     ex_dir = pathlib.Path('examples/') / example
     bsp = pathlib.Path("hw/bsp")
-    
+
     if (bsp / board / "board.mk").exists():
         # board without family
         board_dir = bsp / board
@@ -15,19 +24,19 @@ def skip_example(example, board):
         if not board_dir:
             # Skip unknown boards
             return True
-    
+
         board_dir = list(board_dir)[0]
-    
+
         family_dir = board_dir.parent.parent
         family = family_dir.name
-    
+
         # family CMake
         family_mk = family_dir / "family.cmake"
-    
+
         # family.mk
         if not family_mk.exists():
             family_mk = family_dir / "family.mk"
-    
+
         mk_contents = family_mk.read_text()
 
     # Find the mcu, first in family mk then board mk
@@ -66,3 +75,48 @@ def skip_example(example, board):
                     "family:" + family in onlys)
 
     return False
+
+
+def build_example(example, board):
+    start_time = time.monotonic()
+    flash_size = "-"
+    sram_size = "-"
+
+    # succeeded, failed, skipped
+    ret = [0, 0, 0]
+
+    # Check if board is skipped
+    if skip_example(example, board):
+        status = SKIPPED
+        ret[2] = 1
+        print(build_format.format(example, board, status, '-', flash_size, sram_size))
+    else:
+        build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True,
+                                      stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+        if build_result.returncode == 0:
+            status = SUCCEEDED
+            ret[0] = 1
+            (flash_size, sram_size) = build_size(example, board)
+            subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True,
+                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        else:
+            status = FAILED
+            ret[1] = 1
+
+        build_duration = time.monotonic() - start_time
+        print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
+
+        if build_result.returncode != 0:
+            print(build_result.stdout.decode("utf-8"))
+
+    return ret
+
+
+def build_size(example, board):
+    elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
+    size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
+    size_list = size_output.split('\n')[1].split('\t')
+    flash_size = int(size_list[0])
+    sram_size = int(size_list[1]) + int(size_list[2])
+    return (flash_size, sram_size)

tools/get_dependencies.py

@@ -0,0 +1,25 @@
+import os
+import sys
+import subprocess
+
+
+# dependency lookup (ABC sorted)
+# deps = {
+#    'LPC11UXX' : [ [] ]
+# }
+
+
+def get_family_dep(family):
+    for entry in os.scandir("hw/bsp/{}/boards".format(family)):
+        if entry.is_dir():
+            result = subprocess.run("make -C examples/device/board_test BOARD={} get-deps".format(entry.name),
+                                    shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            print(result.stdout.decode("utf-8"))
+            return result.returncode
+
+status = 0
+all_family = sys.argv[1:]
+for f in all_family:
+    status += get_family_dep(f)
+
+sys.exit(status)