rp2,esp32,extmod: Implement UPDATE_SUBMODULES in CMake.
Rather than having Make calling CMake to generate a list of submodules and
then run a Make target (which is complex and prone to masking other
errors), implement the submodule update logic in CMake itself.
Internal CMake-side changes are that GIT_SUBMODULES is now a CMake list,
and the trigger variable name is changed from ECHO_SUBMODULES to
UPDATE_SUBMODULES.
The run is otherwise 100% a normal CMake run now, so most of the other
special casing can be removed.
Signed-off-by: Angus Gratton <angus@redyak.com.au>
diff --git a/extmod/extmod.cmake b/extmod/extmod.cmake
index 3643f1a..7cacd0a 100644
--- a/extmod/extmod.cmake
+++ b/extmod/extmod.cmake
@@ -107,62 +107,57 @@
if(MICROPY_PY_BTREE)
set(MICROPY_LIB_BERKELEY_DIR "${MICROPY_DIR}/lib/berkeley-db-1.xx")
- string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/berkeley-db-1.xx)
+ list(APPEND GIT_SUBMODULES lib/berkeley-db-1.xx)
- if(ECHO_SUBMODULES)
- # No-op, we're just doing submodule/variant discovery.
- # Cannot run the add_library/target_include_directories rules (even though
- # the build won't run) because IDF will attempt verify the files exist.
- elseif(NOT EXISTS ${MICROPY_LIB_BERKELEY_DIR}/README)
+ if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_BERKELEY_DIR}/README)
# Regular build, submodule not initialised -- fail with a clear error.
message(FATAL_ERROR " MICROPY_PY_BTREE is enabled but the berkeley-db submodule is not initialised.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
- else()
- # Regular build, we have the submodule.
- add_library(micropy_extmod_btree OBJECT
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
- ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
- ${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
- )
-
- target_include_directories(micropy_extmod_btree PRIVATE
- ${MICROPY_LIB_BERKELEY_DIR}/include
- )
-
- if(NOT BERKELEY_DB_CONFIG_FILE)
- set(BERKELEY_DB_CONFIG_FILE "${MICROPY_DIR}/extmod/berkeley-db/berkeley_db_config_port.h")
- endif()
-
- target_compile_definitions(micropy_extmod_btree PRIVATE
- BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
- )
-
- # The include directories and compile definitions below are needed to build
- # modbtree.c and should be added to the main MicroPython target.
-
- list(APPEND MICROPY_INC_CORE
- "${MICROPY_LIB_BERKELEY_DIR}/include"
- )
-
- list(APPEND MICROPY_DEF_CORE
- MICROPY_PY_BTREE=1
- BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
- )
-
- list(APPEND MICROPY_SOURCE_EXTMOD
- ${MICROPY_EXTMOD_DIR}/modbtree.c
- )
endif()
+
+ add_library(micropy_extmod_btree OBJECT
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
+ ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
+ ${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
+ )
+
+ target_include_directories(micropy_extmod_btree PRIVATE
+ ${MICROPY_LIB_BERKELEY_DIR}/include
+ )
+
+ if(NOT BERKELEY_DB_CONFIG_FILE)
+ set(BERKELEY_DB_CONFIG_FILE "${MICROPY_DIR}/extmod/berkeley-db/berkeley_db_config_port.h")
+ endif()
+
+ target_compile_definitions(micropy_extmod_btree PRIVATE
+ BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
+ )
+
+ # The include directories and compile definitions below are needed to build
+ # modbtree.c and should be added to the main MicroPython target.
+
+ list(APPEND MICROPY_INC_CORE
+ "${MICROPY_LIB_BERKELEY_DIR}/include"
+ )
+
+ list(APPEND MICROPY_DEF_CORE
+ MICROPY_PY_BTREE=1
+ BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
+ )
+
+ list(APPEND MICROPY_SOURCE_EXTMOD
+ ${MICROPY_EXTMOD_DIR}/modbtree.c
+ )
endif()
# Library for mbedtls
@@ -350,5 +345,5 @@
${MICROPY_LIB_LWIP_DIR}/include
)
- string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/lwip)
+ list(APPEND GIT_SUBMODULES lib/lwip)
endif()
diff --git a/ports/esp32/Makefile b/ports/esp32/Makefile
index 1ce4d97..a4e5531 100644
--- a/ports/esp32/Makefile
+++ b/ports/esp32/Makefile
@@ -106,12 +106,10 @@
size-files:
$(call RUN_IDF_PY,size-files)
-# Running the build with ECHO_SUBMODULES set will trigger py/mkrules.cmake to
-# print out the value of the GIT_SUBMODULES variable, prefixed with
-# "GIT_SUBMODULES", and then abort. This extracts out that line from the idf.py
-# output and passes the list of submodules to py/mkrules.mk which does the
-# `git submodule init` on each.
+# Run idf.py with the UPDATE_SUBMODULES flag to update
+# necessary submodules for this board.
+#
+# This is done in a dedicated build directory as some CMake cache values are not
+# set correctly if not all submodules are loaded yet.
submodules:
- @GIT_SUBMODULES=$$(IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D ECHO_SUBMODULES=1 build 2>&1 | \
- grep '^GIT_SUBMODULES=' | cut -d= -f2); \
- $(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules
+ IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D UPDATE_SUBMODULES=1 reconfigure
diff --git a/ports/esp32/esp32_common.cmake b/ports/esp32/esp32_common.cmake
index 6473f04..f7b0090 100644
--- a/ports/esp32/esp32_common.cmake
+++ b/ports/esp32/esp32_common.cmake
@@ -71,8 +71,8 @@
${MICROPY_DIR}/drivers/dht/dht.c
)
-string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/tinyusb)
-if(MICROPY_PY_TINYUSB AND NOT ECHO_SUBMODULES)
+list(APPEND GIT_SUBMODULES lib/tinyusb)
+if(MICROPY_PY_TINYUSB)
set(TINYUSB_SRC "${MICROPY_DIR}/lib/tinyusb/src")
string(TOUPPER OPT_MCU_${IDF_TARGET} tusb_mcu)
@@ -195,6 +195,17 @@
set(MICROPY_LDFRAGMENTS ${MICROPY_USER_LDFRAGMENTS})
endif()
+if (UPDATE_SUBMODULES)
+ # ESP-IDF checks if some paths exist before CMake does. Some paths don't
+ # yet exist if this is an UPDATE_SUBMODULES pass on a brand new checkout, so remove
+ # any path which might not exist yet. A "real" build will not set UPDATE_SUBMODULES.
+ unset(MICROPY_SOURCE_TINYUSB)
+ unset(MICROPY_SOURCE_EXTMOD)
+ unset(MICROPY_SOURCE_LIB)
+ unset(MICROPY_INC_TINYUSB)
+ unset(MICROPY_INC_CORE)
+endif()
+
# Register the main IDF component.
idf_component_register(
SRCS
diff --git a/ports/rp2/CMakeLists.txt b/ports/rp2/CMakeLists.txt
index 7002ad8..f89e279 100644
--- a/ports/rp2/CMakeLists.txt
+++ b/ports/rp2/CMakeLists.txt
@@ -76,8 +76,8 @@
endif()
# Necessary submodules for all boards.
-string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/mbedtls)
-string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/tinyusb)
+list(APPEND GIT_SUBMODULES lib/mbedtls)
+list(APPEND GIT_SUBMODULES lib/tinyusb)
# Include component cmake fragments
include(${MICROPY_DIR}/py/py.cmake)
@@ -340,7 +340,7 @@
endif()
if (MICROPY_BLUETOOTH_BTSTACK)
- string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/btstack)
+ list(APPEND GIT_SUBMODULES lib/btstack)
list(APPEND MICROPY_SOURCE_PORT mpbtstackport.c)
@@ -358,8 +358,8 @@
endif()
if(MICROPY_BLUETOOTH_NIMBLE)
- string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/mynewt-nimble)
- if(NOT (${ECHO_SUBMODULES}) AND NOT EXISTS ${MICROPY_DIR}/lib/mynewt-nimble/nimble/host/include/host/ble_hs.h)
+ list(APPEND GIT_SUBMODULES lib/mynewt-nimble)
+ if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/mynewt-nimble/nimble/host/include/host/ble_hs.h)
message(FATAL_ERROR " mynewt-nimble not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
endif()
@@ -386,8 +386,8 @@
)
if (MICROPY_PY_NETWORK_CYW43)
- string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/cyw43-driver)
- if((NOT (${ECHO_SUBMODULES})) AND NOT EXISTS ${MICROPY_DIR}/lib/cyw43-driver/src/cyw43.h)
+ list(APPEND GIT_SUBMODULES lib/cyw43-driver)
+ if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/cyw43-driver/src/cyw43.h)
message(FATAL_ERROR " cyw43-driver not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
endif()
@@ -433,8 +433,8 @@
endif()
if (MICROPY_PY_NETWORK_WIZNET5K)
- string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/wiznet5k)
- if((NOT (${ECHO_SUBMODULES})) AND NOT EXISTS ${MICROPY_DIR}/lib/wiznet5k/README.md)
+ list(APPEND GIT_SUBMODULES lib/wiznet5k)
+ if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/wiznet5k/README.md)
message(FATAL_ERROR " wiznet5k not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
endif()
diff --git a/ports/rp2/Makefile b/ports/rp2/Makefile
index 200899d..bfc85f3 100644
--- a/ports/rp2/Makefile
+++ b/ports/rp2/Makefile
@@ -65,15 +65,11 @@
clean:
$(RM) -rf $(BUILD)
-# First ensure that pico-sdk is initialised, then use cmake to pick everything
-# else (including board-specific dependencies).
-# Running the build with ECHO_SUBMODULES set will trigger py/mkrules.cmake to
-# print out the value of the GIT_SUBMODULES variable, prefixed with
-# "GIT_SUBMODULES", and then abort. This extracts out that line from the cmake
-# output and passes the list of submodules to py/mkrules.mk which does the
-# `git submodule init` on each.
+# First ensure that pico-sdk is initialised, then run CMake with the
+# UPDATE_SUBMODULES flag to update necessary submodules for this board.
+#
+# This is done in a dedicated build directory as some CMake cache values are not
+# set correctly if not all submodules are loaded yet.
submodules:
$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="lib/pico-sdk" submodules
- @GIT_SUBMODULES=$$(cmake -B $(BUILD)/submodules -DECHO_SUBMODULES=1 ${CMAKE_ARGS} -S . 2>&1 | \
- grep '^GIT_SUBMODULES=' | cut -d= -f2); \
- $(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules
+ cmake -S . -B $(BUILD)/submodules -DUPDATE_SUBMODULES=1 ${CMAKE_ARGS}
diff --git a/py/mkrules.cmake b/py/mkrules.cmake
index cafcbce..4374b8b 100644
--- a/py/mkrules.cmake
+++ b/py/mkrules.cmake
@@ -209,16 +209,11 @@
# Note: target_compile_definitions already added earlier.
if(NOT MICROPY_LIB_DIR)
- string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/micropython-lib)
+ list(APPEND GIT_SUBMODULES lib/micropython-lib)
set(MICROPY_LIB_DIR ${MICROPY_DIR}/lib/micropython-lib)
endif()
- if(ECHO_SUBMODULES)
- # No-op, we're just doing submodule/variant discovery.
- # Note: All the following rules are safe to run in discovery mode even
- # though the submodule might not be available as they do not directly depend
- # on anything from the submodule.
- elseif(NOT EXISTS ${MICROPY_LIB_DIR}/README.md)
+ if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_DIR}/README.md)
message(FATAL_ERROR " micropython-lib not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
endif()
@@ -272,12 +267,29 @@
)
endif()
-# Update submodules
-if(ECHO_SUBMODULES)
- # If cmake is run with GIT_SUBMODULES defined on command line, process the port / board
- # settings then print the final GIT_SUBMODULES variable and exit.
- # Note: the GIT_SUBMODULES is done via echo rather than message, as message splits
- # the output onto multiple lines
- execute_process(COMMAND ${CMAKE_COMMAND} -E echo "GIT_SUBMODULES=${GIT_SUBMODULES}")
- message(FATAL_ERROR "Done")
+# Update submodules, this is invoked on some ports via 'make submodules'.
+#
+# Note: This logic has a Makefile equivalent in py/mkrules.mk
+if(UPDATE_SUBMODULES AND GIT_SUBMODULES)
+ macro(run_git)
+ execute_process(COMMAND git ${ARGV} WORKING_DIRECTORY ${MICROPY_DIR}
+ RESULT_VARIABLE RES)
+ endmacro()
+
+ list(JOIN GIT_SUBMODULES " " GIT_SUBMODULES_MSG)
+ message("Updating submodules: ${GIT_SUBMODULES_MSG}")
+ run_git(submodule sync ${GIT_SUBMODULES})
+ if(RES EQUAL 0)
+ # If available, do blobless partial clones of submodules to save time and space.
+ # A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.).
+ run_git(submodule update --init --filter=blob:none ${GIT_SUBMODULES})
+ # Fallback to standard submodule update if blobless isn't available (earlier than git 2.36.0)
+ if (NOT RES EQUAL 0)
+ run_git(submodule update --init ${GIT_SUBMODULES})
+ endif()
+ endif()
+
+ if (NOT RES EQUAL 0)
+ message(FATAL_ERROR "Submodule update failed")
+ endif()
endif()
diff --git a/py/mkrules.mk b/py/mkrules.mk
index ad855c0..495d8d4 100644
--- a/py/mkrules.mk
+++ b/py/mkrules.mk
@@ -262,19 +262,14 @@
# If available, do blobless partial clones of submodules to save time and space.
# A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.).
# Fallback to standard submodule update if blobless isn't available (earlier than 2.36.0)
+#
+# Note: This target has a CMake equivalent in py/mkrules.cmake
submodules:
$(ECHO) "Updating submodules: $(GIT_SUBMODULES)"
ifneq ($(GIT_SUBMODULES),)
$(Q)cd $(TOP) && git submodule sync $(GIT_SUBMODULES)
$(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) || \
git submodule update --init $(GIT_SUBMODULES)
-else
-ifeq ($(GIT_SUBMODULES_FAIL_IF_EMPTY),1)
- # If you see this error, it may mean the internal step run by the port's build
- # system to find git submodules has failed. Double-check dependencies are set correctly.
- $(ECHO) "Internal build error: The submodule list should not be empty."
- exit 1
-endif
endif
.PHONY: submodules