Implement HW protection mutex

sys_check_timeouts may call other LWIP calls...but we're stuck in the
main LWIP processing loop already!

cores/rp2040/lwip_wrap.cpp

@@ -30,55 +30,16 @@
 #include <pico/mutex.h>
 #include <sys/lock.h>
 #include "_xoshiro.h"
+#include "lwip_wrap.h"
 
-extern void ethernet_arch_lwip_begin() __attribute__((weak));
-extern void ethernet_arch_lwip_end() __attribute__((weak));
-extern void ethernet_arch_lwip_gpio_mask() __attribute__((weak));
-extern void ethernet_arch_lwip_gpio_unmask() __attribute__((weak));
-
-auto_init_recursive_mutex(__lwipMutex); // Only for case with no Ethernet or PicoW, but still doing LWIP (PPP?)
-
-class LWIPMutex {
-public:
-    LWIPMutex() {
-        if (ethernet_arch_lwip_gpio_mask)  {
-            ethernet_arch_lwip_gpio_mask();
-        }
-#if defined(ARDUINO_RASPBERRY_PI_PICO_W)
-        if (rp2040.isPicoW()) {
-            cyw43_arch_lwip_begin();
-            return;
-        }
-#endif
-        if (ethernet_arch_lwip_begin) {
-            ethernet_arch_lwip_begin();
-        } else {
-            recursive_mutex_enter_blocking(&__lwipMutex);
-        }
-    }
-
-    ~LWIPMutex() {
-#if defined(ARDUINO_RASPBERRY_PI_PICO_W)
-        if (rp2040.isPicoW()) {
-            cyw43_arch_lwip_end();
-        } else {
-#endif
-            if (ethernet_arch_lwip_end) {
-                ethernet_arch_lwip_end();
-            } else {
-                recursive_mutex_exit(&__lwipMutex);
-            }
-#if defined(ARDUINO_RASPBERRY_PI_PICO_W)
-        }
-#endif
-        if (ethernet_arch_lwip_gpio_unmask) {
-            ethernet_arch_lwip_gpio_unmask();
-        }
-    }
-};
+//auto_init_recursive_mutex(__lwipMutex); // Only for case with no Ethernet or PicoW, but still doing LWIP (PPP?)
+recursive_mutex_t __lwipMutex;
 
 extern "C" {
 
+    extern void __lwip(__lwip_op op, void *req) __attribute((weak));
+    extern bool __isLWIPThread();
+
     static XoshiroCpp::Xoshiro256PlusPlus *_lwip_rng = nullptr;
     // Random number generator for LWIP
     unsigned long __lwip_rand() {
@@ -89,6 +50,7 @@ extern "C" {
     extern void __real_lwip_init();
     void __wrap_lwip_init() {
         if (!_lwip_rng) {
+            recursive_mutex_init(&__lwipMutex);
             _lwip_rng = new XoshiroCpp::Xoshiro256PlusPlus(micros() * rp2040.getCycleCount());
             __real_lwip_init();
         }
@@ -97,215 +59,404 @@ extern "C" {
     extern u8_t __real_pbuf_header(struct pbuf *p, s16_t header_size);
     u8_t __wrap_pbuf_header(struct pbuf *p, s16_t header_size) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            u8_t ret;
+            __pbuf_header_req req = { p, header_size, &ret };
+            __lwip(__pbuf_header, &req);
+            return ret;
+        }
         return __real_pbuf_header(p, header_size);
     }
 
     extern u8_t __real_pbuf_free(struct pbuf *p);
     u8_t __wrap_pbuf_free(struct pbuf *p) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            u8_t ret;
+            __pbuf_free_req req = { p, &ret };
+            __lwip(__pbuf_free, &req);
+            return ret;
+        }
         return __real_pbuf_free(p);
     }
 
     extern struct pbuf *__real_pbuf_alloc(pbuf_layer l, u16_t length, pbuf_type type);
     struct pbuf *__wrap_pbuf_alloc(pbuf_layer l, u16_t length, pbuf_type type) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            struct pbuf *ret;
+            __pbuf_alloc_req req = {l, length, type, &ret };
+            __lwip(__pbuf_alloc, &req);
+            return ret;
+        }
         return __real_pbuf_alloc(l, length, type);
     }
 
     extern err_t __real_pbuf_take(struct pbuf *buf, const void *dataptr, u16_t len);
     err_t __wrap_pbuf_take(struct pbuf *buf, const void *dataptr, u16_t len) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __pbuf_take_req req = { buf, dataptr, len, &ret };
+            __lwip(__pbuf_take, &req);
+            return ret;
+        }
         return __real_pbuf_take(buf, dataptr, len);
     }
 
     extern u16_t __real_pbuf_copy_partial(const struct pbuf *p, void *dataptr, u16_t len, u16_t offset);
     u16_t __wrap_pbuf_copy_partial(const struct pbuf *p, void *dataptr, u16_t len, u16_t offset) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            u16_t ret;
+            __pbuf_copy_partial_req req = { p, dataptr, len, offset, &ret };
+            __lwip(__pbuf_copy_partial, &req);
+            return ret;
+        }
         return __real_pbuf_copy_partial(p, dataptr, len, offset);
     }
 
     extern void __real_pbuf_ref(struct pbuf *p);
     void __wrap_pbuf_ref(struct pbuf *p) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __pbuf_ref_req req = { p };
+            __lwip(__pbuf_ref, &req);
+            return;
+        }
         __real_pbuf_ref(p);
     }
 
     extern u8_t __real_pbuf_get_at(const struct pbuf* p, u16_t offset);
     u8_t __wrap_pbuf_get_at(const struct pbuf* p, u16_t offset) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            u8_t ret;
+            __pbuf_get_at_req req = { p, offset, &ret };
+            __lwip(__pbuf_get_at, &req);
+            return ret;
+        }
         return __real_pbuf_get_at(p, offset);
     }
 
     extern void *__real_pbuf_get_contiguous(const struct pbuf *p, void *buffer, size_t bufsize, u16_t len, u16_t offset);
     void *__wrap_pbuf_get_contiguous(const struct pbuf *p, void *buffer, size_t bufsize, u16_t len, u16_t offset) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            void *ret;
+            __pbuf_get_contiguous_req req = { p, buffer, bufsize, len, offset, &ret };
+            __lwip(__pbuf_get_contiguous, &req);
+            return ret;
+        }
         return __real_pbuf_get_contiguous(p, buffer, bufsize, len, offset);
     }
 
     extern void __real_pbuf_cat(struct pbuf *head, struct pbuf *tail);
     void __wrap_pbuf_cat(struct pbuf *head, struct pbuf *tail) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __pbuf_cat_req req = { head, tail };
+            __lwip(__pbuf_cat, &req);
+            return;
+        }
         __real_pbuf_cat(head, tail);
     }
 
     extern void __real_tcp_arg(struct tcp_pcb *pcb, void *arg);
     void __wrap_tcp_arg(struct tcp_pcb *pcb, void *arg) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_arg_req req = { pcb, arg };
+            __lwip(__tcp_arg, &req);
+            return;
+        }
         __real_tcp_arg(pcb, arg);
     }
 
     extern struct tcp_pcb *__real_tcp_new(void);
     struct tcp_pcb *__wrap_tcp_new(void) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            struct tcp_pcb *ret;
+            __tcp_new_req req = { &ret };
+            __lwip(__tcp_new, &req);
+            return ret;
+        }
         return __real_tcp_new();
     }
 
     extern err_t __real_tcp_bind(struct tcp_pcb *pcb, ip_addr_t *ipaddr, u16_t port);
     err_t __wrap_tcp_bind(struct tcp_pcb *pcb, ip_addr_t *ipaddr, u16_t port) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __tcp_bind_req req = { pcb, ipaddr, port, &ret };
+            __lwip(__tcp_bind, &req);
+            return ret;
+        }
         return __real_tcp_bind(pcb, ipaddr, port);
     }
 
-    extern struct tcp_pcb *__real_tcp_listen(struct tcp_pcb *pcb);
-    struct tcp_pcb *__wrap_tcp_listen(struct tcp_pcb *pcb) {
-        LWIPMutex m;
-        return __real_tcp_listen(pcb);
-    }
-
     extern struct tcp_pcb *__real_tcp_listen_with_backlog(struct tcp_pcb *pcb, u8_t backlog);
     struct tcp_pcb *__wrap_tcp_listen_with_backlog(struct tcp_pcb *pcb, u8_t backlog) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            struct tcp_pcb *ret;
+            __tcp_listen_with_backlog_req req = { pcb, backlog, &ret };
+            __lwip(__tcp_listen_with_backlog, &req);
+            return ret;
+        }
         return __real_tcp_listen_with_backlog(pcb, backlog);
     }
 
     extern void __real_tcp_accept(struct tcp_pcb *pcb, err_t (* accept)(void *arg, struct tcp_pcb *newpcb, err_t err));
     void __wrap_tcp_accept(struct tcp_pcb *pcb, err_t (* accept)(void *arg, struct tcp_pcb *newpcb, err_t err)) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_accept_req req = { pcb, accept };
+            __lwip(__tcp_accept, &req);
+            return;
+        }
         __real_tcp_accept(pcb, accept);
     }
 
     extern err_t __real_tcp_connect(struct tcp_pcb *pcb, ip_addr_t *ipaddr, u16_t port, err_t (* connected)(void *arg, struct tcp_pcb *tpcb, err_t err));
     err_t __wrap_tcp_connect(struct tcp_pcb *pcb, ip_addr_t *ipaddr, u16_t port, err_t (* connected)(void *arg, struct tcp_pcb *tpcb, err_t err)) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __tcp_connect_req req = { pcb, ipaddr, port, connected, &ret };
+            __lwip(__tcp_connect, &req);
+            return ret;
+        }
         return __real_tcp_connect(pcb, ipaddr, port, connected);
     }
 
     extern err_t __real_tcp_write(struct tcp_pcb *pcb, const void *dataptr, u16_t len, u8_t apiflags);
     err_t __wrap_tcp_write(struct tcp_pcb *pcb, const void *dataptr, u16_t len, u8_t apiflags) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __tcp_write_req req = { pcb, dataptr, len, apiflags, &ret };
+            __lwip(__tcp_write, &req);
+            return ret;
+        }
         return __real_tcp_write(pcb, dataptr, len, apiflags);
     }
 
     extern void __real_tcp_sent(struct tcp_pcb *pcb, err_t (* sent)(void *arg, struct tcp_pcb *tpcb, u16_t len));
     void __wrap_tcp_sent(struct tcp_pcb *pcb, err_t (* sent)(void *arg, struct tcp_pcb *tpcb, u16_t len)) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_sent_req req = { pcb, sent };
+            __lwip(__tcp_sent, &req);
+            return;
+        }
         __real_tcp_sent(pcb, sent);
     }
 
     extern void __real_tcp_recv(struct tcp_pcb *pcb, err_t (* recv)(void *arg, struct tcp_pcb *tpcb, struct pbuf *p, err_t err));
     void __wrap_tcp_recv(struct tcp_pcb *pcb, err_t (* recv)(void *arg, struct tcp_pcb *tpcb, struct pbuf *p, err_t err)) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_recv_req req = { pcb, recv };
+            __lwip(__tcp_recv, &req);
+            return;
+        }
         __real_tcp_recv(pcb, recv);
     }
 
     extern void __real_tcp_recved(struct tcp_pcb *pcb, u16_t len);
     void __wrap_tcp_recved(struct tcp_pcb *pcb, u16_t len) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_recved_req req = { pcb, len };
+            __lwip(__tcp_recved, &req);
+            return;
+        }
         __real_tcp_recved(pcb, len);
     }
 
     extern void __real_tcp_poll(struct tcp_pcb *pcb, err_t (* poll)(void *arg, struct tcp_pcb *tpcb), u8_t interval);
     void __wrap_tcp_poll(struct tcp_pcb *pcb, err_t (* poll)(void *arg, struct tcp_pcb *tpcb), u8_t interval) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_poll_req req = { pcb, poll, interval };
+            __lwip(__tcp_poll, &req);
+            return;
+        }
         __real_tcp_poll(pcb, poll, interval);
     }
 
     extern err_t __real_tcp_close(struct tcp_pcb *pcb);
     err_t __wrap_tcp_close(struct tcp_pcb *pcb) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __tcp_close_req req = { pcb, &ret };
+            __lwip(__tcp_close, &req);
+            return ret;
+        }
         return __real_tcp_close(pcb);
     }
 
     extern void __real_tcp_abort(struct tcp_pcb *pcb);
     void __wrap_tcp_abort(struct tcp_pcb *pcb) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_abort_req req = { pcb };
+            __lwip(__tcp_abort, &req);
+            return;
+        }
         __real_tcp_abort(pcb);
     }
 
     extern void __real_tcp_err(struct tcp_pcb *pcb, void (* err)(void *arg, err_t err));
     void __wrap_tcp_err(struct tcp_pcb *pcb, void (* err)(void *arg, err_t err)) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_err_req req = { pcb, err };
+            __lwip(__tcp_err, &req);
+            return;
+        }
         __real_tcp_err(pcb, err);
     }
 
     extern err_t __real_tcp_output(struct tcp_pcb *pcb);
     err_t __wrap_tcp_output(struct tcp_pcb *pcb) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __tcp_output_req req = { pcb, &ret };
+            __lwip(__tcp_output, &req);
+            return ret;
+        }
         return __real_tcp_output(pcb);
     }
 
     extern void __real_tcp_setprio(struct tcp_pcb *pcb, u8_t prio);
     void __wrap_tcp_setprio(struct tcp_pcb *pcb, u8_t prio) {
         LWIPMutex m;
-        return __real_tcp_setprio(pcb, prio);
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_setprio_req req = { pcb, prio };
+            __lwip(__tcp_setprio, &req);
+            return;
+        }
+        __real_tcp_setprio(pcb, prio);
     }
 
     extern void __real_tcp_backlog_delayed(struct tcp_pcb* pcb);
     void __wrap_tcp_backlog_delayed(struct tcp_pcb* pcb) {
         LWIPMutex m;
-        return __real_tcp_backlog_delayed(pcb);
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_backlog_delayed_req req = { pcb };
+            __lwip(__tcp_backlog_delayed, &req);
+            return;
+        }
+        __real_tcp_backlog_delayed(pcb);
     }
 
     extern void __real_tcp_backlog_accepted(struct tcp_pcb* pcb);
     void __wrap_tcp_backlog_accepted(struct tcp_pcb* pcb) {
         LWIPMutex m;
-        return __real_tcp_backlog_accepted(pcb);
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __tcp_backlog_accepted_req req = { pcb };
+            __lwip(__tcp_backlog_accepted, &req);
+            return;
+        }
+        __real_tcp_backlog_accepted(pcb);
     }
     extern struct udp_pcb *__real_udp_new(void);
     struct udp_pcb *__wrap_udp_new(void) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            struct udp_pcb *ret;
+            __udp_new_req req = { &ret };
+            __lwip(__udp_new, &req);
+            return ret;
+        }
         return __real_udp_new();
     }
 
     extern void __real_udp_remove(struct udp_pcb *pcb);
     void __wrap_udp_remove(struct udp_pcb *pcb) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __udp_remove_req req = { pcb };
+            __lwip(__udp_remove, &req);
+            return;
+        }
         __real_udp_remove(pcb);
     }
 
     extern err_t __real_udp_bind(struct udp_pcb *pcb, ip_addr_t *ipaddr, u16_t port);
     err_t __wrap_udp_bind(struct udp_pcb *pcb, ip_addr_t *ipaddr, u16_t port) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __udp_bind_req req = { pcb, ipaddr, port, &ret };
+            __lwip(__udp_bind, &req);
+            return ret;
+        }
         return __real_udp_bind(pcb, ipaddr, port);
     }
 
     extern err_t __real_udp_connect(struct udp_pcb *pcb, ip_addr_t *ipaddr, u16_t port);
     err_t __wrap_udp_connect(struct udp_pcb *pcb, ip_addr_t *ipaddr, u16_t port) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __udp_connect_req req = { pcb, ipaddr, port, &ret };
+            __lwip(__udp_connect, &req);
+            return ret;
+        }
         return __real_udp_connect(pcb, ipaddr, port);
     }
 
     extern err_t __real_udp_disconnect(struct udp_pcb *pcb);
     err_t __wrap_udp_disconnect(struct udp_pcb *pcb) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __udp_disconnect_req req = { pcb, &ret };
+            __lwip(__udp_disconnect, &req);
+            return ret;
+        }
         return __real_udp_disconnect(pcb);
     }
 
     extern err_t __real_udp_send(struct udp_pcb *pcb, struct pbuf *p);
     err_t __wrap_udp_send(struct udp_pcb *pcb, struct pbuf *p) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __udp_send_req req = { pcb, p, &ret };
+            __lwip(__udp_send, &req);
+            return ret;
+        }
         return __real_udp_send(pcb, p);
     }
 
     extern void __real_udp_recv(struct udp_pcb *pcb, void (* recv)(void *arg, struct udp_pcb *upcb, struct pbuf *p, ip_addr_t *addr, u16_t port), void *recv_arg);
     void __wrap_udp_recv(struct udp_pcb *pcb, void (* recv)(void *arg, struct udp_pcb *upcb, struct pbuf *p, ip_addr_t *addr, u16_t port), void *recv_arg) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __udp_recv_req req = { pcb, recv, recv_arg };
+            __lwip(__udp_recv, &req);
+            return;
+        }
         __real_udp_recv(pcb, recv, recv_arg);
     }
 
     extern err_t __real_udp_sendto_if(struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *dst_ip, u16_t dst_port, struct netif *netif);
     err_t __wrap_udp_sendto_if(struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *dst_ip, u16_t dst_port, struct netif *netif) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __udp_sendto_if_req req = { pcb, p, dst_ip, dst_port, netif, &ret };
+            __lwip(__udp_sendto_if, &req);
+            return ret;
+        }
         return __real_udp_sendto_if(pcb, p, dst_ip, dst_port, netif);
     }
 
@@ -313,60 +464,115 @@ extern "C" {
     extern void __real_sys_check_timeouts();
     void __wrap_sys_check_timeouts(void) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __lwip(__sys_check_timeouts, nullptr);
+            return;
+        }
         __real_sys_check_timeouts();
     }
 
     extern err_t __real_dns_gethostbyname(const char *hostname, ip_addr_t *addr, dns_found_callback found, void *callback_arg);
     err_t __wrap_dns_gethostbyname(const char *hostname, ip_addr_t *addr, dns_found_callback found, void *callback_arg) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __dns_gethostbyname_req req = { hostname, addr, found, callback_arg, &ret };
+            __lwip(__dns_gethostbyname, &req);
+            return ret;
+        }
         return __real_dns_gethostbyname(hostname, addr, found, callback_arg);
     }
 
     extern err_t __real_dns_gethostbyname_addrtype(const char *hostname, ip_addr_t *addr, dns_found_callback found, void *callback_arg, u8_t dns_addrtype);
     err_t __wrap_dns_gethostbyname_addrtype(const char *hostname, ip_addr_t *addr, dns_found_callback found, void *callback_arg, u8_t dns_addrtype) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __dns_gethostbyname_addrtype_req req = { hostname, addr, found, callback_arg, dns_addrtype, &ret };
+            __lwip(__dns_gethostbyname_addrtype, &req);
+            return ret;
+        }
         return __real_dns_gethostbyname_addrtype(hostname, addr, found, callback_arg, dns_addrtype);
     }
 
     extern struct raw_pcb *__real_raw_new(u8_t proto);
     struct raw_pcb *__wrap_raw_new(u8_t proto) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            struct raw_pcb *ret;
+            __raw_new_req req = { proto, &ret };
+            __lwip(__raw_new, &req);
+            return ret;
+        }
         return __real_raw_new(proto);
     }
 
     extern void __real_raw_recv(struct raw_pcb *pcb, raw_recv_fn recv, void *recv_arg);
     void __wrap_raw_recv(struct raw_pcb *pcb, raw_recv_fn recv, void *recv_arg) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __raw_recv_req req = { pcb, recv, recv_arg };
+            __lwip(__raw_recv, &req);
+            return;
+        }
         __real_raw_recv(pcb, recv, recv_arg);
     }
 
     extern err_t __real_raw_bind(struct raw_pcb *pcb, const ip_addr_t *ipaddr);
     err_t __wrap_raw_bind(struct raw_pcb *pcb, const ip_addr_t *ipaddr) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __raw_bind_req req = { pcb, ipaddr, &ret };
+            __lwip(__raw_bind, &req);
+            return ret;
+        }
         return __real_raw_bind(pcb, ipaddr);
     }
 
     extern err_t __real_raw_sendto(struct raw_pcb *pcb, struct pbuf *p, const ip_addr_t *ipaddr);
     err_t __wrap_raw_sendto(struct raw_pcb *pcb, struct pbuf *p, const ip_addr_t *ipaddr) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            err_t ret;
+            __raw_sendto_req req = { pcb, p, ipaddr, &ret };
+            __lwip(__raw_sendto, &req);
+            return ret;
+        }
         return __real_raw_sendto(pcb, p, ipaddr);
     }
 
     extern void __real_raw_remove(struct raw_pcb *pcb);
     void __wrap_raw_remove(struct raw_pcb *pcb) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __raw_remove_req req = { pcb };
+            __lwip(__raw_remove, &req);
+            return;
+        }
         __real_raw_remove(pcb);
     }
 
     extern struct netif *__real_netif_add(struct netif *netif, const ip4_addr_t *ipaddr, const ip4_addr_t *netmask, const ip4_addr_t *gw, void *state, netif_init_fn init, netif_input_fn input);
     struct netif *__wrap_netif_add(struct netif *netif, const ip4_addr_t *ipaddr, const ip4_addr_t *netmask, const ip4_addr_t *gw, void *state, netif_init_fn init, netif_input_fn input) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            struct netif *ret;
+            __netif_add_req req = { netif, ipaddr, netmask, gw, state, init, input, &ret };
+            __lwip(__netif_add, &req);
+            return ret;
+        }
         return __real_netif_add(netif, ipaddr, netmask, gw, state, init, input);
     }
 
     extern void __real_netif_remove(struct netif *netif);
     void __wrap_netif_remove(struct netif *netif) {
         LWIPMutex m;
+        if (__isFreeRTOS && !__isLWIPThread()) {
+            __netif_remove_req req = { netif };
+            __lwip(__netif_remove, &req);
+            return;
+        }
         __real_netif_remove(netif);
     }
 

cores/rp2040/lwip_wrap.h

@@ -0,0 +1,470 @@
+/*
+    FreeRTOS LWIP wrappers, implement a single LWIP work task
+
+    Copyright (c) 2024 Earle F. Philhower, III <earlephilhower@yahoo.com>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+#pragma once
+
+#include <Arduino.h>
+#include <lwip/pbuf.h>
+#include <lwip/udp.h>
+#include <lwip/tcp.h>
+#include <lwip/dns.h>
+#include <lwip/raw.h>
+#include <lwip/timeouts.h>
+
+
+extern void ethernet_arch_lwip_begin() __attribute__((weak));
+extern void ethernet_arch_lwip_end() __attribute__((weak));
+extern void ethernet_arch_lwip_gpio_mask() __attribute__((weak));
+extern void ethernet_arch_lwip_gpio_unmask() __attribute__((weak));
+
+//auto_init_recursive_mutex(__lwipMutex); // Only for case with no Ethernet or PicoW, but still doing LWIP (PPP?)
+extern recursive_mutex_t __lwipMutex;
+
+
+class LWIPMutex {
+public:
+    LWIPMutex() {
+        if (ethernet_arch_lwip_gpio_mask)  {
+            ethernet_arch_lwip_gpio_mask();
+        }
+#if defined(ARDUINO_RASPBERRY_PI_PICO_W)
+        if (rp2040.isPicoW()) {
+            cyw43_arch_lwip_begin();
+            return;
+        }
+#endif
+        if (ethernet_arch_lwip_begin) {
+            ethernet_arch_lwip_begin();
+        } else {
+            recursive_mutex_enter_blocking(&__lwipMutex);
+        }
+    }
+
+    ~LWIPMutex() {
+#if defined(ARDUINO_RASPBERRY_PI_PICO_W)
+        if (rp2040.isPicoW()) {
+            cyw43_arch_lwip_end();
+        } else {
+#endif
+            if (ethernet_arch_lwip_end) {
+                ethernet_arch_lwip_end();
+            } else {
+                recursive_mutex_exit(&__lwipMutex);
+            }
+#if defined(ARDUINO_RASPBERRY_PI_PICO_W)
+        }
+#endif
+        if (ethernet_arch_lwip_gpio_unmask) {
+            ethernet_arch_lwip_gpio_unmask();
+        }
+    }
+};
+
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Implement all LWIP operations in a single FreeRTOS task.  Calls to LWIP will
+// actually post work on the work queue and wait until the LWIP task indicates
+// completion
+
+// Enumerated type for LWIP request
+typedef enum {
+    __lwip_init = 1000,
+
+    __pbuf_header = 2000,
+    __pbuf_free,
+    __pbuf_alloc,
+    __pbuf_take,
+    __pbuf_copy_partial,
+    __pbuf_ref,
+    __pbuf_get_at,
+    __pbuf_get_contiguous,
+    __pbuf_cat,
+
+    __tcp_arg = 3000,
+    __tcp_new,
+    __tcp_bind,
+    __tcp_listen,
+    __tcp_listen_with_backlog,
+    __tcp_accept,
+    __tcp_connect,
+    __tcp_write,
+    __tcp_sent,
+    __tcp_recv,
+    __tcp_recved,
+    __tcp_poll,
+    __tcp_close,
+    __tcp_abort,
+    __tcp_err,
+    __tcp_output,
+    __tcp_setprio,
+    __tcp_backlog_delayed,
+    __tcp_backlog_accepted,
+
+    __udp_new = 4000,
+    __udp_remove,
+    __udp_bind,
+    __udp_connect,
+    __udp_disconnect,
+    __udp_send,
+    __udp_recv,
+    __udp_sendto_if,
+
+    __sys_check_timeouts = 5000,
+
+    __dns_gethostbyname = 6000,
+    __dns_gethostbyname_addrtype,
+
+    __raw_new = 7000,
+    __raw_recv,
+    __raw_bind,
+    __raw_sendto,
+    __raw_remove,
+
+    __netif_add = 8000,
+    __netif_remove
+} __lwip_op;
+
+// Set up a local request buffer and call this to add to lwip work queue.  Will only return once lwip operation completed
+// LWIP callbacks will happen from the LWIP task at some future time
+void __lwip(__lwip_op op, void *req);
+void __lwipISR(__lwip_op op, void *req);
+
+extern void __real_lwip_init();
+extern u8_t __real_pbuf_header(struct pbuf *p, s16_t header_size);
+extern u8_t __real_pbuf_free(struct pbuf *p);
+extern struct pbuf *__real_pbuf_alloc(pbuf_layer l, u16_t length, pbuf_type type);
+extern err_t __real_pbuf_take(struct pbuf *buf, const void *dataptr, u16_t len);
+extern u16_t __real_pbuf_copy_partial(const struct pbuf *p, void *dataptr, u16_t len, u16_t offset);
+extern void __real_pbuf_ref(struct pbuf *p);
+extern u8_t __real_pbuf_get_at(const struct pbuf* p, u16_t offset);
+extern void *__real_pbuf_get_contiguous(const struct pbuf *p, void *buffer, size_t bufsize, u16_t len, u16_t offset);
+extern void __real_pbuf_cat(struct pbuf *head, struct pbuf *tail);
+extern void __real_tcp_arg(struct tcp_pcb *pcb, void *arg);
+extern struct tcp_pcb *__real_tcp_new(void);
+extern err_t __real_tcp_bind(struct tcp_pcb *pcb, ip_addr_t *ipaddr, u16_t port);
+extern struct tcp_pcb *__real_tcp_listen(struct tcp_pcb *pcb);
+extern struct tcp_pcb *__real_tcp_listen_with_backlog(struct tcp_pcb *pcb, u8_t backlog);
+extern void __real_tcp_accept(struct tcp_pcb *pcb, err_t (* accept)(void *arg, struct tcp_pcb *newpcb, err_t err));
+extern err_t __real_tcp_connect(struct tcp_pcb *pcb, ip_addr_t *ipaddr, u16_t port, err_t (* connected)(void *arg, struct tcp_pcb *tpcb, err_t err));
+extern err_t __real_tcp_write(struct tcp_pcb *pcb, const void *dataptr, u16_t len, u8_t apiflags);
+extern void __real_tcp_sent(struct tcp_pcb *pcb, err_t (* sent)(void *arg, struct tcp_pcb *tpcb, u16_t len));
+extern void __real_tcp_recv(struct tcp_pcb *pcb, err_t (* recv)(void *arg, struct tcp_pcb *tpcb, struct pbuf *p, err_t err));
+extern void __real_tcp_recved(struct tcp_pcb *pcb, u16_t len);
+extern void __real_tcp_poll(struct tcp_pcb *pcb, err_t (* poll)(void *arg, struct tcp_pcb *tpcb), u8_t interval);
+extern err_t __real_tcp_close(struct tcp_pcb *pcb);
+extern void __real_tcp_abort(struct tcp_pcb *pcb);
+extern void __real_tcp_err(struct tcp_pcb *pcb, void (* err)(void *arg, err_t err));
+extern err_t __real_tcp_output(struct tcp_pcb *pcb);
+extern void __real_tcp_setprio(struct tcp_pcb *pcb, u8_t prio);
+extern void __real_tcp_backlog_delayed(struct tcp_pcb* pcb);
+extern void __real_tcp_backlog_accepted(struct tcp_pcb* pcb);
+extern struct udp_pcb *__real_udp_new(void);
+extern void __real_udp_remove(struct udp_pcb *pcb);
+extern err_t __real_udp_bind(struct udp_pcb *pcb, ip_addr_t *ipaddr, u16_t port);
+extern err_t __real_udp_connect(struct udp_pcb *pcb, ip_addr_t *ipaddr, u16_t port);
+extern err_t __real_udp_disconnect(struct udp_pcb *pcb);
+extern err_t __real_udp_send(struct udp_pcb *pcb, struct pbuf *p);
+extern void __real_udp_recv(struct udp_pcb *pcb, void (* recv)(void *arg, struct udp_pcb *upcb, struct pbuf *p, ip_addr_t *addr, u16_t port), void *recv_arg);
+extern err_t __real_udp_sendto_if(struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *dst_ip, u16_t dst_port, struct netif *netif);
+extern void __real_sys_check_timeouts();
+extern err_t __real_dns_gethostbyname(const char *hostname, ip_addr_t *addr, dns_found_callback found, void *callback_arg);
+extern err_t __real_dns_gethostbyname_addrtype(const char *hostname, ip_addr_t *addr, dns_found_callback found, void *callback_arg, u8_t dns_addrtype);
+extern struct raw_pcb *__real_raw_new(u8_t proto);
+extern void __real_raw_recv(struct raw_pcb *pcb, raw_recv_fn recv, void *recv_arg);
+extern err_t __real_raw_bind(struct raw_pcb *pcb, const ip_addr_t *ipaddr);
+extern err_t __real_raw_sendto(struct raw_pcb *pcb, struct pbuf *p, const ip_addr_t *ipaddr);
+extern void __real_raw_remove(struct raw_pcb *pcb);
+extern struct netif *__real_netif_add(struct netif *netif, const ip4_addr_t *ipaddr, const ip4_addr_t *netmask, const ip4_addr_t *gw, void *state, netif_init_fn init, netif_input_fn input);
+extern void __real_netif_remove(struct netif *netif);
+
+typedef struct {
+    struct pbuf *p;
+    s16_t header_size;
+    u8_t *ret;
+} __pbuf_header_req;
+
+typedef struct {
+    struct pbuf *p;
+    u8_t *ret;
+} __pbuf_free_req;
+
+typedef struct {
+    pbuf_layer l;
+    u16_t length;
+    pbuf_type type;
+    struct pbuf **ret;
+} __pbuf_alloc_req;
+
+typedef struct {
+    struct pbuf *buf;
+    const void *dataptr;
+    u16_t len;
+    err_t *ret;
+} __pbuf_take_req;
+
+typedef struct {
+    const struct pbuf *p;
+    void *dataptr;
+    u16_t len;
+    u16_t offset;
+    u16_t *ret;
+} __pbuf_copy_partial_req;
+
+typedef struct {
+    struct pbuf *p;
+} __pbuf_ref_req;
+
+typedef struct {
+    const struct pbuf* p;
+    u16_t offset;
+    u8_t *ret;
+} __pbuf_get_at_req;
+
+typedef struct {
+    const struct pbuf *p;
+    void *buffer;
+    size_t bufsize;
+    u16_t len;
+    u16_t offset;
+    void **ret;
+} __pbuf_get_contiguous_req;
+
+typedef struct {
+    struct pbuf *head;
+    struct pbuf *tail;
+} __pbuf_cat_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    void *arg;
+} __tcp_arg_req;
+
+typedef struct {
+    struct tcp_pcb **ret;
+} __tcp_new_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    ip_addr_t *ipaddr;
+    u16_t port;
+    err_t *ret;
+} __tcp_bind_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    struct tcp_pcb **ret;
+} __tcp_listen_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    u8_t backlog;
+    struct tcp_pcb **ret;
+} __tcp_listen_with_backlog_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    err_t (* accept)(void *arg, struct tcp_pcb *newpcb, err_t err);
+} __tcp_accept_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    ip_addr_t *ipaddr;
+    u16_t port;
+    err_t (* connected)(void *arg, struct tcp_pcb *tpcb, err_t err);
+    err_t *ret;
+} __tcp_connect_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    const void *dataptr;
+    u16_t len;
+    u8_t apiflags;
+    err_t *ret;
+} __tcp_write_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    err_t (* sent)(void *arg, struct tcp_pcb *tpcb, u16_t len);
+} __tcp_sent_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    err_t (* recv)(void *arg, struct tcp_pcb *tpcb, struct pbuf *p, err_t err);
+} __tcp_recv_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    u16_t len;
+} __tcp_recved_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    err_t (* poll)(void *arg, struct tcp_pcb *tpcb);
+    u8_t interval;
+} __tcp_poll_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    err_t *ret;
+} __tcp_close_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+} __tcp_abort_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    void (* err)(void *arg, err_t err);
+} __tcp_err_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    err_t *ret;
+} __tcp_output_req;
+
+typedef struct {
+    struct tcp_pcb *pcb;
+    u8_t prio;
+} __tcp_setprio_req;
+
+typedef struct {
+    struct tcp_pcb* pcb;
+} __tcp_backlog_delayed_req;
+
+typedef struct {
+    struct tcp_pcb* pcb;
+} __tcp_backlog_accepted_req;
+
+typedef struct {
+    struct udp_pcb **ret;
+} __udp_new_req;
+
+typedef struct {
+    struct udp_pcb *pcb;
+} __udp_remove_req;
+
+typedef struct {
+    struct udp_pcb *pcb;
+    ip_addr_t *ipaddr;
+    u16_t port;
+    err_t *ret;
+} __udp_bind_req;
+
+typedef struct {
+    struct udp_pcb *pcb;
+    ip_addr_t *ipaddr;
+    u16_t port;
+    err_t *ret;
+} __udp_connect_req;
+
+typedef struct {
+    struct udp_pcb *pcb;
+    err_t *ret;
+} __udp_disconnect_req;
+
+typedef struct {
+    struct udp_pcb *pcb;
+    struct pbuf *p;
+    err_t *ret;
+} __udp_send_req;
+
+typedef struct {
+    struct udp_pcb *pcb;
+    void (* recv)(void *arg, struct udp_pcb *upcb, struct pbuf *p, ip_addr_t *addr, u16_t port);
+    void *recv_arg;
+} __udp_recv_req;
+
+typedef struct {
+    struct udp_pcb *pcb;
+    struct pbuf *p;
+    const ip_addr_t *dst_ip;
+    u16_t dst_port;
+    struct netif *netif;
+    err_t *ret;
+} __udp_sendto_if_req;
+
+typedef struct {
+    const char *hostname;
+    ip_addr_t *addr;
+    dns_found_callback found;
+    void *callback_arg;
+    err_t *ret;
+} __dns_gethostbyname_req;
+
+typedef struct {
+     const char *hostname;
+     ip_addr_t *addr;
+     dns_found_callback found;
+     void *callback_arg;
+     u8_t dns_addrtype;
+     err_t *ret;
+} __dns_gethostbyname_addrtype_req;
+
+typedef struct {
+    u8_t proto;
+    struct raw_pcb **ret;
+} __raw_new_req;
+
+typedef struct {
+    struct raw_pcb *pcb;
+    raw_recv_fn recv;
+    void *recv_arg;
+} __raw_recv_req;
+
+typedef struct {
+    struct raw_pcb *pcb;
+    const ip_addr_t *ipaddr;
+    err_t *ret;
+} __raw_bind_req;
+
+typedef struct {
+    struct raw_pcb *pcb;
+    struct pbuf *p;
+    const ip_addr_t *ipaddr;
+    err_t *ret;
+} __raw_sendto_req;
+
+typedef struct {
+    struct raw_pcb *pcb;
+} __raw_remove_req;
+
+typedef struct {
+    struct netif *netif;
+    const ip4_addr_t *ipaddr;
+    const ip4_addr_t *netmask;
+    const ip4_addr_t *gw;
+    void *state;
+    netif_init_fn init;
+    netif_input_fn input;
+    struct netif **ret;
+} __netif_add_req;
+
+typedef struct {
+    struct netif *netif;
+} __netif_remove_req;
+
+
+#ifdef __cplusplus
+};
+#endif

include/lwipopts.h

@@ -38,6 +38,8 @@ extern unsigned long __lwip_rand(void);
 #define TCP_MSS                     1460
 #define TCP_SND_BUF                 (8 * TCP_MSS)
 #define TCP_SND_QUEUELEN            ((4 * (TCP_SND_BUF) + (TCP_MSS - 1)) / (TCP_MSS))
+#define TCP_LISTEN_BACKLOG          1
+#define TCP_DEFAULT_LISTEN_BACKLOG  2
 #define LWIP_NETIF_STATUS_CALLBACK  1
 #define LWIP_NETIF_LINK_CALLBACK    1
 #define LWIP_NETIF_HOSTNAME         1

lib/platform_wrap.txt

@@ -166,7 +166,7 @@
 
 -Wl,--wrap=tcp_arg
 -Wl,--wrap=tcp_new
--Wl,--wrap=tcp_listen
+-Wl,--wrap=tcp_bind
 -Wl,--wrap=tcp_listen_with_backlog
 -Wl,--wrap=tcp_accept
 -Wl,--wrap=tcp_connect

libraries/FreeRTOS/src/FreeRTOSConfig.h

@@ -26,6 +26,7 @@
 #define configSUPPORT_STATIC_ALLOCATION   1
 #define configSTACK_DEPTH_TYPE            uint32_t
 #define configUSE_TASK_PREEMPTION_DISABLE 1
+#define configTASK_NOTIFICATION_ARRAY_ENTRIES 4
 
 #define configUSE_NEWLIB_REENTRANT        1
 #define configNEWLIB_REENTRANT_IS_DYNAMIC 1

libraries/FreeRTOS/src/variantHooks.cpp

@@ -193,6 +193,354 @@ extern "C" void __no_inline_not_in_flash_func(__freertos_resume_other_core)() {
     vTaskPreemptionEnable(nullptr);
 }
 
+#include <lwip_wrap.h>
+
+static TaskHandle_t __lwipTask;
+static QueueHandle_t __lwipQueue;
+typedef struct {
+    __lwip_op op;
+    void *req;
+    TaskHandle_t wakeup;
+} LWIPWork;
+
+#define TASK_NOTIFY_LWIP_WAKEUP (configTASK_NOTIFICATION_ARRAY_ENTRIES - 1)
+
+extern "C" void __lwip(__lwip_op op, void *req) {
+    LWIPWork w;
+    TaskStatus_t t;
+    vTaskGetInfo(nullptr, &t, pdFALSE, eInvalid); // TODO - can we speed this up???
+    w.op = op;
+    w.req = req;
+    w.wakeup = t.xHandle;
+    if (!xQueueSend(__lwipQueue, &w, portMAX_DELAY)) {
+        panic("LWIP task send failed");
+    }
+    ulTaskNotifyTakeIndexed(TASK_NOTIFY_LWIP_WAKEUP, pdTRUE, portMAX_DELAY);
+}
+
+extern "C" bool __isLWIPThread() {
+    TaskStatus_t t;
+    vTaskGetInfo(nullptr, &t, pdFALSE, eInvalid); // TODO - can we speed this up???
+    return t.xHandle == __lwipTask;
+}
+
+
+static void lwipThread(void *params) {
+    (void) params;
+    LWIPWork w;
+    assert(__isLWIPThread());
+
+    while (true) {
+        if (xQueueReceive(__lwipQueue, &w, portMAX_DELAY)) {
+            switch (w.op) {
+                case __lwip_init:
+                {
+                    __real_lwip_init();
+                    break;
+                }
+                case __pbuf_header:
+                {
+                    __pbuf_header_req *r = (__pbuf_header_req *)w.req;
+                    *(r->ret) = __real_pbuf_header(r->p, r->header_size);
+                    break;
+                }
+                case __pbuf_free:
+                {
+                    __pbuf_free_req *r = (__pbuf_free_req *)w.req;
+                    *(r->ret) = __real_pbuf_free(r->p);
+                    break;
+                }
+                case __pbuf_alloc:
+                {
+                    __pbuf_alloc_req *r = (__pbuf_alloc_req *)w.req;
+                    *(r->ret) = __real_pbuf_alloc(r->l, r->length, r->type);
+                    break;
+                }
+                case __pbuf_take:
+                {
+                    __pbuf_take_req *r = (__pbuf_take_req *)w.req;
+                    *(r->ret) = __real_pbuf_take(r->buf, r->dataptr, r->len);
+                    break;
+                }
+                case __pbuf_copy_partial:
+                {
+                    __pbuf_copy_partial_req *r = (__pbuf_copy_partial_req *)w.req;
+                    *(r->ret) = __real_pbuf_copy_partial(r->p, r->dataptr, r->len, r->offset);
+                    break;
+                }
+                case __pbuf_ref:
+                {
+                    __pbuf_ref_req *r = (__pbuf_ref_req *)w.req;
+                    __real_pbuf_ref(r->p);
+                    break;
+                }
+                case __pbuf_get_at:
+                {
+                    __pbuf_get_at_req *r = (__pbuf_get_at_req *)w.req;
+                    *(r->ret) = __real_pbuf_get_at(r->p, r->offset);
+                    break;
+                }
+                case __pbuf_get_contiguous:
+                {
+                    __pbuf_get_contiguous_req *r = (__pbuf_get_contiguous_req *)w.req;
+                    *(r->ret) = __real_pbuf_get_contiguous(r->p, r->buffer, r->bufsize, r->len, r->offset);
+                    break;
+                }
+                case __pbuf_cat:
+                {
+                    __pbuf_cat_req *r = (__pbuf_cat_req *)w.req;
+                    __real_pbuf_cat(r->head, r->tail);
+                    break;
+                }
+                case __tcp_arg:
+                {
+                    __tcp_arg_req *r = (__tcp_arg_req *)w.req;
+                    __real_tcp_arg(r->pcb, r->arg);
+                    break;
+                }
+                case __tcp_new:
+                {
+                    __tcp_new_req *r = (__tcp_new_req *)w.req;
+                    *(r->ret) = __real_tcp_new();
+                    break;
+                }
+                case __tcp_bind:
+                {
+                    __tcp_bind_req *r = (__tcp_bind_req *)w.req;
+                    *(r->ret) = __real_tcp_bind(r->pcb, r->ipaddr, r->port);
+                    break;
+                }
+                case __tcp_listen_with_backlog:
+                {
+                    __tcp_listen_with_backlog_req *r = (__tcp_listen_with_backlog_req *)w.req;
+                    *(r->ret) = __real_tcp_listen_with_backlog(r->pcb, r->backlog);
+                    break;
+                }
+                case __tcp_accept:
+                {
+                    __tcp_accept_req *r = (__tcp_accept_req *)w.req;
+                    __real_tcp_accept(r->pcb, r->accept);
+                    break;
+                }
+                case __tcp_connect:
+                {
+                    __tcp_connect_req *r = (__tcp_connect_req *)w.req;
+                    *(r->ret) = __real_tcp_connect(r->pcb, r->ipaddr, r->port, r->connected);
+                    break;
+                }
+                case __tcp_write:
+                {
+                    __tcp_write_req *r = (__tcp_write_req *)w.req;
+                    *(r->ret) = __real_tcp_write(r->pcb, r->dataptr, r->len, r->apiflags);
+                    break;
+                }
+                case __tcp_sent:
+                {
+                    __tcp_sent_req *r = (__tcp_sent_req *)w.req;
+                    __real_tcp_sent(r->pcb, r->sent);
+                    break;
+                }
+                case __tcp_recv:
+                {
+                    __tcp_recv_req *r = (__tcp_recv_req *)w.req;
+                    __real_tcp_recv(r->pcb, r->recv);
+                    break;
+                }
+                case __tcp_recved:
+                {
+                    __tcp_recved_req *r = (__tcp_recved_req *)w.req;
+                    __real_tcp_recved(r->pcb, r->len);
+                    break;
+                }
+                case __tcp_poll:
+                {
+                    __tcp_poll_req *r = (__tcp_poll_req *)w.req;
+                    __real_tcp_poll(r->pcb, r->poll, r->interval);
+                    break;
+                }
+                case __tcp_close:
+                {
+                    __tcp_close_req *r = (__tcp_close_req *)w.req;
+                    *(r->ret) = __real_tcp_close(r->pcb);
+                    break;
+                }
+                case __tcp_abort:
+                {
+                    __tcp_abort_req *r = (__tcp_abort_req *)w.req;
+                    __real_tcp_abort(r->pcb);
+                    break;
+                }
+                case __tcp_err:
+                {
+                    __tcp_err_req *r = (__tcp_err_req *)w.req;
+                    __real_tcp_err(r->pcb, r->err);
+                    break;
+                }
+                case __tcp_output:
+                {
+                    __tcp_output_req *r = (__tcp_output_req *)w.req;
+                    *(r->ret) = __real_tcp_output(r->pcb);
+                    break;
+                }
+                case __tcp_setprio:
+                {
+                    __tcp_setprio_req *r = (__tcp_setprio_req *)w.req;
+                    __real_tcp_setprio(r->pcb, r->prio);
+                    break;
+                }
+                case __tcp_backlog_delayed:
+                {
+                    __tcp_backlog_delayed_req *r = (__tcp_backlog_delayed_req *)w.req;
+                    __real_tcp_backlog_delayed(r->pcb);
+                    break;
+                }
+                case __tcp_backlog_accepted:
+                {
+                    __tcp_backlog_accepted_req *r = (__tcp_backlog_accepted_req *)w.req;
+                    __real_tcp_backlog_accepted(r->pcb);
+                    break;
+                }
+                case __udp_new:
+                {
+                    __udp_new_req *r = (__udp_new_req *)w.req;
+                    *(r->ret) = __real_udp_new();
+                    break;
+                }
+                case  __udp_remove:
+                {
+                    __udp_remove_req *r = (__udp_remove_req *)w.req;
+                    __real_udp_remove(r->pcb);
+                    break;
+                }
+                case __udp_bind:
+                {
+                    __udp_bind_req *r = (__udp_bind_req *)w.req;
+                    *(r->ret) = __real_udp_bind(r->pcb, r->ipaddr, r->port);
+                    break;
+                }
+                case __udp_connect:
+                {
+                    __udp_connect_req *r = (__udp_connect_req *)w.req;
+                    *(r->ret) = __real_udp_connect(r->pcb, r->ipaddr, r->port);
+                    break;
+                }
+                case __udp_disconnect:
+                {
+                    __udp_disconnect_req *r = (__udp_disconnect_req *)w.req;
+                    *(r->ret) = __real_udp_disconnect(r->pcb);
+                    break;
+                }
+                case __udp_send:
+                {
+                    __udp_send_req *r = (__udp_send_req *)w.req;
+                    *(r->ret) = __real_udp_send(r->pcb, r->p);
+                    break;
+                }
+                case __udp_recv:
+                {
+                    __udp_recv_req *r = (__udp_recv_req *)w.req;
+                    __real_udp_recv(r->pcb, r->recv, r->recv_arg);
+                    break;
+                }
+                case __udp_sendto_if:
+                {
+                    __udp_sendto_if_req *r = (__udp_sendto_if_req *)w.req;
+                    *(r->ret) = __real_udp_sendto_if(r->pcb, r->p, r->dst_ip, r->dst_port, r->netif);
+                    break;
+                }
+                case __sys_check_timeouts:
+                {
+                    __real_sys_check_timeouts();
+                    break;
+                }
+                case __dns_gethostbyname:
+                {
+                    __dns_gethostbyname_req *r = (__dns_gethostbyname_req *)w.req;
+                    *(r->ret) = __real_dns_gethostbyname(r->hostname, r->addr, r->found, r->callback_arg);
+                    break;
+                }
+                case __dns_gethostbyname_addrtype:
+                {
+                    __dns_gethostbyname_addrtype_req *r = (__dns_gethostbyname_addrtype_req *)w.req;
+                    *(r->ret) = __real_dns_gethostbyname_addrtype(r->hostname, r->addr, r->found, r->callback_arg, r->dns_addrtype);
+                    break;
+                }
+                case __raw_new:
+                {
+                    __raw_new_req *r = (__raw_new_req *)w.req;
+                    *(r->ret) = __real_raw_new(r->proto);
+                    break;
+                }
+                case __raw_recv:
+                {
+                    __raw_recv_req *r = (__raw_recv_req *)w.req;
+                    __real_raw_recv(r->pcb, r->recv, r->recv_arg);
+                    break;
+                }
+                case __raw_bind:
+                {
+                    __raw_bind_req *r = (__raw_bind_req *)w.req;
+                    *(r->ret) = __real_raw_bind(r->pcb, r->ipaddr);
+                    break;
+                }
+                case __raw_sendto:
+                {
+                    __raw_sendto_req *r = (__raw_sendto_req *)w.req;
+                    *(r->ret) = __real_raw_sendto(r->pcb, r->p, r->ipaddr);
+                    break;
+                }
+                case __raw_remove:
+                {
+                    __raw_remove_req *r = (__raw_remove_req *)w.req;
+                    __real_raw_remove(r->pcb);
+                    break;
+                }
+                case __netif_add:
+                {
+                    __netif_add_req *r = (__netif_add_req *)w.req;
+                    *(r->ret) = __real_netif_add(r->netif, r->ipaddr, r->netmask, r->gw, r->state, r->init, r->input);
+                    break;
+                }
+                case __netif_remove:
+                {
+                    __netif_remove_req *r = (__netif_remove_req *)w.req;
+                    __real_netif_remove(r->netif);
+                    break;
+                }
+                default:
+                {
+                    // Any new unimplemented calls = ERROR!!!
+                    panic("Unimplemented LWIP thread action");
+                    break;
+                }
+            }
+            // Work done, return value set, just tickle the calling task
+            xTaskNotifyGiveIndexed(w.wakeup, TASK_NOTIFY_LWIP_WAKEUP);
+        }
+    }
+}
+
+
+
+extern "C" void ethernet_timeout_reached(void *context, void *worker);
+static TaskHandle_t __ethernetTask;
+SemaphoreHandle_t __hwMutex;
+
+static void ethernetTask(void *param) {
+    (void) param;
+    while (true) {
+        delay(20);
+        {
+            LWIPMutex m;
+            ethernet_timeout_reached(nullptr, nullptr);
+        }
+    }
+}
+
+void __startEthernetTask() {
+    xTaskCreate(ethernetTask, "Ethernet", 1024, 0, configMAX_PRIORITIES / 2 - 1, &__ethernetTask);
+    vTaskCoreAffinitySet(__ethernetTask, 1 << 0);
+}
 
 extern mutex_t __usb_mutex;
 static TaskHandle_t __usbTask;
@@ -216,6 +564,12 @@ void startFreeRTOS(void) {
     xTaskCreate(IdleThisCore, "IdleCore1", 128, 0, configMAX_PRIORITIES - 1, __idleCoreTask + 1);
     vTaskCoreAffinitySet(__idleCoreTask[1], 1 << 1);
 
+    // LWIP runs on core 0 only
+    __lwipQueue = xQueueCreate(16, sizeof(LWIPWork));
+    __hwMutex = xSemaphoreCreateMutex();
+    xTaskCreate(lwipThread, "LWIP", 1024, 0, configMAX_PRIORITIES / 2 - 1, &__lwipTask);
+    vTaskCoreAffinitySet(__lwipTask, 1 << 0);
+
     // Initialise and run the freeRTOS scheduler. Execution should never return here.
     __freeRTOSinitted = true;
     vTaskStartScheduler();

libraries/lwIP_Ethernet/src/LwipEthernet.cpp

@@ -170,8 +170,8 @@ static async_context_t *lwip_ethernet_init_default_async_context(void) {
 uint32_t __ethernet_timeout_reached_calls = 0;
 static uint32_t _pollingPeriod = 20;
 // This will only be called under the protection of the async context mutex, so no re-entrancy checks needed
-static void ethernet_timeout_reached(__unused async_context_t *context, __unused async_at_time_worker_t *worker) {
-    assert(worker == &ethernet_timeout_worker);
+extern "C" void ethernet_timeout_reached(__unused async_context_t *context, __unused async_at_time_worker_t *worker) {
+//    assert(worker == &ethernet_timeout_worker);
     __ethernet_timeout_reached_calls++;
     ethernet_arch_lwip_gpio_mask(); // Ensure non-polled devices won't interrupt us
     for (auto handlePacket : _handlePacketList) {
@@ -192,6 +192,7 @@ static void update_next_timeout(async_context_t *context, async_when_pending_wor
     worker->work_pending = true;
     async_context_add_at_time_worker_in_ms(context, &ethernet_timeout_worker, _pollingPeriod);
 }
+extern void __startEthernetTask() __attribute((weak));
 
 void __startEthernetContext() {
     if (__ethernetContextInitted) {
@@ -209,7 +210,11 @@ void __startEthernetContext() {
     ethernet_timeout_worker.do_work = ethernet_timeout_reached;
     always_pending_update_timeout_worker.work_pending = true;
     always_pending_update_timeout_worker.do_work = update_next_timeout;
-    async_context_add_when_pending_worker(_context, &always_pending_update_timeout_worker);
+    if (__isFreeRTOS) {
+        __startEthernetTask();
+    } else {
+        async_context_add_when_pending_worker(_context, &always_pending_update_timeout_worker);
+    }
     __ethernetContextInitted = true;
 }
 

libraries/lwIP_Ethernet/src/LwipIntfDev.h

@@ -40,6 +40,9 @@
 #include <lwip/inet_chksum.h>
 #include <lwip/apps/sntp.h>
 
+#include <_freertos.h>
+extern SemaphoreHandle_t __hwMutex;
+
 
 #include "SPI.h"
 #include "LwipIntf.h"
@@ -369,9 +372,12 @@ bool LwipIntfDev<RawDev>::begin(const uint8_t* macAddress, const uint16_t mtu) {
         return false;
     }
 
+    __freertos_mutex_take(__hwMutex);
     if (!RawDev::begin(_macAddress, &_netif)) {
+        __freertos_mutex_give(__hwMutex);
         return false;
     }
+    __freertos_mutex_give(__hwMutex);
 
     if (_intrPin < 0) {
         _phID = __addEthernetPacketHandler([this] { this->handlePackets(); });
@@ -435,7 +441,9 @@ void LwipIntfDev<RawDev>::end() {
             __removeEthernetGPIO(_intrPin);
         }
 
+        __freertos_mutex_take(__hwMutex);
         RawDev::end();
+        __freertos_mutex_give(__hwMutex);
 
         netif_remove(&_netif);
 
@@ -466,7 +474,9 @@ template<class RawDev>
 err_t LwipIntfDev<RawDev>::linkoutput_s(netif* netif, struct pbuf* pbuf) {
     LwipIntfDev* lid = (LwipIntfDev*)netif->state;
     ethernet_arch_lwip_begin();
+    __freertos_mutex_take(__hwMutex);
     uint16_t len = lid->sendFrame((const uint8_t*)pbuf->payload, pbuf->len);
+    __freertos_mutex_give(__hwMutex);
     lid->_packetsSent++;
 #if PHY_HAS_CAPTURE
     if (phy_capture) {
@@ -550,7 +560,9 @@ err_t LwipIntfDev<RawDev>::handlePackets() {
             return ERR_OK;
         }
 
+        __freertos_mutex_take(__hwMutex);
         uint16_t tot_len = RawDev::readFrameSize();
+        __freertos_mutex_give(__hwMutex);
         if (!tot_len) {
             return ERR_OK;
         }
@@ -568,11 +580,15 @@ err_t LwipIntfDev<RawDev>::handlePackets() {
             if (pbuf) {
                 pbuf_free(pbuf);
             }
+            __freertos_mutex_take(__hwMutex);
             RawDev::discardFrame(tot_len);
+            __freertos_mutex_give(__hwMutex);
             return ERR_BUF;
         }
 
+        __freertos_mutex_take(__hwMutex);
         uint16_t len = RawDev::readFrameData((uint8_t*)pbuf->payload, tot_len);
+        __freertos_mutex_give(__hwMutex);
         if (len != tot_len) {
             // tot_len is given by readFrameSize()
             // and is supposed to be honoured by readFrameData()

tools/libpico/CMakeLists.txt

@@ -36,7 +36,7 @@ target_compile_definitions(common INTERFACE
 
 target_compile_options(common INTERFACE
     -fno-exceptions
-    -Os
+    -Og
     $<$<COMPILE_LANGUAGE:CXX>:-fno-rtti>
 )
 

tools/libpico/lwipopts.h

@@ -38,6 +38,8 @@ extern unsigned long __lwip_rand(void);
 #define TCP_MSS                     1460
 #define TCP_SND_BUF                 (8 * TCP_MSS)
 #define TCP_SND_QUEUELEN            ((4 * (TCP_SND_BUF) + (TCP_MSS - 1)) / (TCP_MSS))
+#define TCP_LISTEN_BACKLOG          1
+#define TCP_DEFAULT_LISTEN_BACKLOG  2
 #define LWIP_NETIF_STATUS_CALLBACK  1
 #define LWIP_NETIF_LINK_CALLBACK    1
 #define LWIP_NETIF_HOSTNAME         1