diff options
author | Peter Maydell <peter.maydell@linaro.org> | 2019-03-25 18:15:43 +0000 |
---|---|---|
committer | Peter Maydell <peter.maydell@linaro.org> | 2019-03-25 18:15:43 +0000 |
commit | 7e9a2137ce3bdae00cb864b5f14d7880aa699ad0 (patch) | |
tree | 0dadc2a74d588ab9eac1aa8f25b37c92615c15c9 /migration | |
parent | d132baa05ed7a647067c807fd452ec794cc2ecb7 (diff) | |
parent | c38c1c142e64901b09f5ac000695071ed8c0e3a5 (diff) | |
download | qemu-7e9a2137ce3bdae00cb864b5f14d7880aa699ad0.zip |
Merge remote-tracking branch 'remotes/juanquintela/tags/migration-pull-request' into staging
Pull request
- Rebase last pull request
- Drop multifd
- several other minor fixesLaLaLa
# gpg: Signature made Mon 25 Mar 2019 17:46:29 GMT
# gpg: using RSA key F487EF185872D723
# gpg: Good signature from "Juan Quintela <quintela@redhat.com>" [full]
# gpg: aka "Juan Quintela <quintela@trasno.org>" [full]
# Primary key fingerprint: 1899 FF8E DEBF 58CC EE03 4B82 F487 EF18 5872 D723
* remotes/juanquintela/tags/migration-pull-request:
migration/postcopy: Update the bandwidth during postcopy
Migration/colo.c: Make user obtain the last COLO mode info after failover
Migration/colo.c: Add the necessary checks for colo_do_failover
Migration/colo.c: Add new COLOExitReason to handle all failover state
Migration/colo.c: Fix COLO failover status error
migration/rdma: Check qemu_rdma_init_one_block
migration: add support for a "tls-authz" migration parameter
multifd: Drop x-
multifd: Add some padding
multifd: Change default packet size
multifd: Be flexible about packet size
multifd: Drop x-multifd-page-count parameter
multifd: Create new next_packet_size field
multifd: Rename "size" member to pages_alloc
multifd: Only send pages when packet are not empty
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
Diffstat (limited to 'migration')
-rw-r--r-- | migration/colo.c | 57 | ||||
-rw-r--r-- | migration/migration.c | 70 | ||||
-rw-r--r-- | migration/migration.h | 1 | ||||
-rw-r--r-- | migration/ram.c | 82 | ||||
-rw-r--r-- | migration/rdma.c | 6 | ||||
-rw-r--r-- | migration/tls.c | 2 | ||||
-rw-r--r-- | migration/trace-events | 4 |
7 files changed, 135 insertions, 87 deletions
diff --git a/migration/colo.c b/migration/colo.c index 5ba610dc01..238a6d62c7 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -38,6 +38,9 @@ static bool vmstate_loading; static Notifier packets_compare_notifier; +/* User need to know colo mode after COLO failover */ +static COLOMode last_colo_mode; + #define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024) bool migration_in_colo_state(void) @@ -121,6 +124,7 @@ static void secondary_vm_do_failover(void) } /* Notify COLO incoming thread that failover work is finished */ qemu_sem_post(&mis->colo_incoming_sem); + /* For Secondary VM, jump to incoming co */ if (mis->migration_incoming_co) { qemu_coroutine_enter(mis->migration_incoming_co); @@ -196,10 +200,16 @@ void colo_do_failover(MigrationState *s) vm_stop_force_state(RUN_STATE_COLO); } - if (get_colo_mode() == COLO_MODE_PRIMARY) { + switch (get_colo_mode()) { + case COLO_MODE_PRIMARY: primary_vm_do_failover(); - } else { + break; + case COLO_MODE_SECONDARY: secondary_vm_do_failover(); + break; + default: + error_report("colo_do_failover failed because the colo mode" + " could not be obtained"); } } @@ -257,16 +267,21 @@ COLOStatus *qmp_query_colo_status(Error **errp) COLOStatus *s = g_new0(COLOStatus, 1); s->mode = get_colo_mode(); + s->last_mode = last_colo_mode; switch (failover_get_state()) { case FAILOVER_STATUS_NONE: s->reason = COLO_EXIT_REASON_NONE; break; - case FAILOVER_STATUS_REQUIRE: + case FAILOVER_STATUS_COMPLETED: s->reason = COLO_EXIT_REASON_REQUEST; break; default: - s->reason = COLO_EXIT_REASON_ERROR; + if (migration_in_colo_state()) { + s->reason = COLO_EXIT_REASON_PROCESSING; + } else { + s->reason = COLO_EXIT_REASON_ERROR; + } } return s; @@ -504,6 +519,12 @@ static void colo_process_checkpoint(MigrationState *s) Error *local_err = NULL; int ret; + last_colo_mode = get_colo_mode(); + if (last_colo_mode != COLO_MODE_PRIMARY) { + error_report("COLO mode must be COLO_MODE_PRIMARY"); + return; + } + failover_init_state(); s->rp_state.from_dst_file = qemu_file_get_return_path(s->to_dst_file); @@ -578,16 +599,13 @@ out: * or the user triggered failover. */ switch (failover_get_state()) { - case FAILOVER_STATUS_NONE: - qapi_event_send_colo_exit(COLO_MODE_PRIMARY, - COLO_EXIT_REASON_ERROR); - break; - case FAILOVER_STATUS_REQUIRE: + case FAILOVER_STATUS_COMPLETED: qapi_event_send_colo_exit(COLO_MODE_PRIMARY, COLO_EXIT_REASON_REQUEST); break; default: - abort(); + qapi_event_send_colo_exit(COLO_MODE_PRIMARY, + COLO_EXIT_REASON_ERROR); } /* Hope this not to be too long to wait here */ @@ -680,6 +698,12 @@ void *colo_process_incoming_thread(void *opaque) migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, MIGRATION_STATUS_COLO); + last_colo_mode = get_colo_mode(); + if (last_colo_mode != COLO_MODE_SECONDARY) { + error_report("COLO mode must be COLO_MODE_SECONDARY"); + return NULL; + } + failover_init_state(); mis->to_src_file = qemu_file_get_return_path(mis->from_src_file); @@ -849,17 +873,18 @@ out: error_report_err(local_err); } + /* + * There are only two reasons we can get here, some error happened + * or the user triggered failover. + */ switch (failover_get_state()) { - case FAILOVER_STATUS_NONE: - qapi_event_send_colo_exit(COLO_MODE_SECONDARY, - COLO_EXIT_REASON_ERROR); - break; - case FAILOVER_STATUS_REQUIRE: + case FAILOVER_STATUS_COMPLETED: qapi_event_send_colo_exit(COLO_MODE_SECONDARY, COLO_EXIT_REASON_REQUEST); break; default: - abort(); + qapi_event_send_colo_exit(COLO_MODE_SECONDARY, + COLO_EXIT_REASON_ERROR); } if (fb) { diff --git a/migration/migration.c b/migration/migration.c index df6fd8e0e5..69f75124c9 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -82,7 +82,6 @@ /* The delay time (in ms) between two COLO checkpoints */ #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100) #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2 -#define DEFAULT_MIGRATE_MULTIFD_PAGE_COUNT 16 /* Background transfer rate for postcopy, 0 means unlimited, note * that page requests can still exceed this limit. @@ -758,6 +757,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) params->tls_creds = g_strdup(s->parameters.tls_creds); params->has_tls_hostname = true; params->tls_hostname = g_strdup(s->parameters.tls_hostname); + params->has_tls_authz = true; + params->tls_authz = g_strdup(s->parameters.tls_authz); params->has_max_bandwidth = true; params->max_bandwidth = s->parameters.max_bandwidth; params->has_downtime_limit = true; @@ -766,10 +767,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) params->x_checkpoint_delay = s->parameters.x_checkpoint_delay; params->has_block_incremental = true; params->block_incremental = s->parameters.block_incremental; - params->has_x_multifd_channels = true; - params->x_multifd_channels = s->parameters.x_multifd_channels; - params->has_x_multifd_page_count = true; - params->x_multifd_page_count = s->parameters.x_multifd_page_count; + params->has_multifd_channels = true; + params->multifd_channels = s->parameters.multifd_channels; params->has_xbzrle_cache_size = true; params->xbzrle_cache_size = s->parameters.xbzrle_cache_size; params->has_max_postcopy_bandwidth = true; @@ -1152,20 +1151,12 @@ static bool migrate_params_check(MigrationParameters *params, Error **errp) /* x_checkpoint_delay is now always positive */ - if (params->has_x_multifd_channels && (params->x_multifd_channels < 1)) { + if (params->has_multifd_channels && (params->multifd_channels < 1)) { error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_channels", "is invalid, it should be in the range of 1 to 255"); return false; } - if (params->has_x_multifd_page_count && - (params->x_multifd_page_count < 1 || - params->x_multifd_page_count > 10000)) { - error_setg(errp, QERR_INVALID_PARAMETER_VALUE, - "multifd_page_count", - "is invalid, it should be in the range of 1 to 10000"); - return false; - } if (params->has_xbzrle_cache_size && (params->xbzrle_cache_size < qemu_target_page_size() || @@ -1274,11 +1265,8 @@ static void migrate_params_test_apply(MigrateSetParameters *params, if (params->has_block_incremental) { dest->block_incremental = params->block_incremental; } - if (params->has_x_multifd_channels) { - dest->x_multifd_channels = params->x_multifd_channels; - } - if (params->has_x_multifd_page_count) { - dest->x_multifd_page_count = params->x_multifd_page_count; + if (params->has_multifd_channels) { + dest->multifd_channels = params->multifd_channels; } if (params->has_xbzrle_cache_size) { dest->xbzrle_cache_size = params->xbzrle_cache_size; @@ -1345,9 +1333,15 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s); } + if (params->has_tls_authz) { + g_free(s->parameters.tls_authz); + assert(params->tls_authz->type == QTYPE_QSTRING); + s->parameters.tls_authz = g_strdup(params->tls_authz->u.s); + } + if (params->has_max_bandwidth) { s->parameters.max_bandwidth = params->max_bandwidth; - if (s->to_dst_file) { + if (s->to_dst_file && !migration_in_postcopy()) { qemu_file_set_rate_limit(s->to_dst_file, s->parameters.max_bandwidth / XFER_LIMIT_RATIO); } @@ -1367,11 +1361,8 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) if (params->has_block_incremental) { s->parameters.block_incremental = params->block_incremental; } - if (params->has_x_multifd_channels) { - s->parameters.x_multifd_channels = params->x_multifd_channels; - } - if (params->has_x_multifd_page_count) { - s->parameters.x_multifd_page_count = params->x_multifd_page_count; + if (params->has_multifd_channels) { + s->parameters.multifd_channels = params->multifd_channels; } if (params->has_xbzrle_cache_size) { s->parameters.xbzrle_cache_size = params->xbzrle_cache_size; @@ -1379,6 +1370,10 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) } if (params->has_max_postcopy_bandwidth) { s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth; + if (s->to_dst_file && migration_in_postcopy()) { + qemu_file_set_rate_limit(s->to_dst_file, + s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO); + } } if (params->has_max_cpu_throttle) { s->parameters.max_cpu_throttle = params->max_cpu_throttle; @@ -2130,7 +2125,7 @@ bool migrate_use_multifd(void) s = migrate_get_current(); - return s->enabled_capabilities[MIGRATION_CAPABILITY_X_MULTIFD]; + return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD]; } bool migrate_pause_before_switchover(void) @@ -2149,16 +2144,7 @@ int migrate_multifd_channels(void) s = migrate_get_current(); - return s->parameters.x_multifd_channels; -} - -int migrate_multifd_page_count(void) -{ - MigrationState *s; - - s = migrate_get_current(); - - return s->parameters.x_multifd_page_count; + return s->parameters.multifd_channels; } int migrate_use_xbzrle(void) @@ -3400,12 +3386,9 @@ static Property migration_properties[] = { DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState, parameters.x_checkpoint_delay, DEFAULT_MIGRATE_X_CHECKPOINT_DELAY), - DEFINE_PROP_UINT8("x-multifd-channels", MigrationState, - parameters.x_multifd_channels, + DEFINE_PROP_UINT8("multifd-channels", MigrationState, + parameters.multifd_channels, DEFAULT_MIGRATE_MULTIFD_CHANNELS), - DEFINE_PROP_UINT32("x-multifd-page-count", MigrationState, - parameters.x_multifd_page_count, - DEFAULT_MIGRATE_MULTIFD_PAGE_COUNT), DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState, parameters.xbzrle_cache_size, DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE), @@ -3440,7 +3423,7 @@ static Property migration_properties[] = { DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM), DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK), DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH), - DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_X_MULTIFD), + DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD), DEFINE_PROP_END_OF_LIST(), }; @@ -3494,8 +3477,7 @@ static void migration_instance_init(Object *obj) params->has_downtime_limit = true; params->has_x_checkpoint_delay = true; params->has_block_incremental = true; - params->has_x_multifd_channels = true; - params->has_x_multifd_page_count = true; + params->has_multifd_channels = true; params->has_xbzrle_cache_size = true; params->has_max_postcopy_bandwidth = true; params->has_max_cpu_throttle = true; diff --git a/migration/migration.h b/migration/migration.h index 99e99e56bd..0f986935e1 100644 --- a/migration/migration.h +++ b/migration/migration.h @@ -274,7 +274,6 @@ bool migrate_auto_converge(void); bool migrate_use_multifd(void); bool migrate_pause_before_switchover(void); int migrate_multifd_channels(void); -int migrate_multifd_page_count(void); int migrate_use_xbzrle(void); int64_t migrate_xbzrle_cache_size(void); diff --git a/migration/ram.c b/migration/ram.c index 35bd6213e9..d7f8fe45a8 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -583,20 +583,29 @@ exit: #define MULTIFD_FLAG_SYNC (1 << 0) +/* This value needs to be a multiple of qemu_target_page_size() */ +#define MULTIFD_PACKET_SIZE (512 * 1024) + typedef struct { uint32_t magic; uint32_t version; unsigned char uuid[16]; /* QemuUUID */ uint8_t id; + uint8_t unused1[7]; /* Reserved for future use */ + uint64_t unused2[4]; /* Reserved for future use */ } __attribute__((packed)) MultiFDInit_t; typedef struct { uint32_t magic; uint32_t version; uint32_t flags; - uint32_t size; - uint32_t used; + /* maximum number of allocated pages */ + uint32_t pages_alloc; + uint32_t pages_used; + /* size of the next packet that contains pages */ + uint32_t next_packet_size; uint64_t packet_num; + uint64_t unused[4]; /* Reserved for future use */ char ramblock[256]; uint64_t offset[]; } __attribute__((packed)) MultiFDPacket_t; @@ -643,6 +652,8 @@ typedef struct { MultiFDPacket_t *packet; /* multifd flags for each packet */ uint32_t flags; + /* size of the next packet that contains pages */ + uint32_t next_packet_size; /* global number of generated multifd packets */ uint64_t packet_num; /* thread local variables */ @@ -679,6 +690,8 @@ typedef struct { /* global number of generated multifd packets */ uint64_t packet_num; /* thread local variables */ + /* size of the next packet that contains pages */ + uint32_t next_packet_size; /* packets sent through this channel */ uint64_t num_packets; /* pages sent through this channel */ @@ -776,13 +789,15 @@ static void multifd_pages_clear(MultiFDPages_t *pages) static void multifd_send_fill_packet(MultiFDSendParams *p) { MultiFDPacket_t *packet = p->packet; + uint32_t page_max = MULTIFD_PACKET_SIZE / qemu_target_page_size(); int i; packet->magic = cpu_to_be32(MULTIFD_MAGIC); packet->version = cpu_to_be32(MULTIFD_VERSION); packet->flags = cpu_to_be32(p->flags); - packet->size = cpu_to_be32(migrate_multifd_page_count()); - packet->used = cpu_to_be32(p->pages->used); + packet->pages_alloc = cpu_to_be32(page_max); + packet->pages_used = cpu_to_be32(p->pages->used); + packet->next_packet_size = cpu_to_be32(p->next_packet_size); packet->packet_num = cpu_to_be64(p->packet_num); if (p->pages->block) { @@ -797,6 +812,7 @@ static void multifd_send_fill_packet(MultiFDSendParams *p) static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp) { MultiFDPacket_t *packet = p->packet; + uint32_t pages_max = MULTIFD_PACKET_SIZE / qemu_target_page_size(); RAMBlock *block; int i; @@ -818,22 +834,35 @@ static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp) p->flags = be32_to_cpu(packet->flags); - packet->size = be32_to_cpu(packet->size); - if (packet->size > migrate_multifd_page_count()) { + packet->pages_alloc = be32_to_cpu(packet->pages_alloc); + /* + * If we recevied a packet that is 100 times bigger than expected + * just stop migration. It is a magic number. + */ + if (packet->pages_alloc > pages_max * 100) { error_setg(errp, "multifd: received packet " - "with size %d and expected maximum size %d", - packet->size, migrate_multifd_page_count()) ; + "with size %d and expected a maximum size of %d", + packet->pages_alloc, pages_max * 100) ; return -1; } + /* + * We received a packet that is bigger than expected but inside + * reasonable limits (see previous comment). Just reallocate. + */ + if (packet->pages_alloc > p->pages->allocated) { + multifd_pages_clear(p->pages); + multifd_pages_init(packet->pages_alloc); + } - p->pages->used = be32_to_cpu(packet->used); - if (p->pages->used > packet->size) { + p->pages->used = be32_to_cpu(packet->pages_used); + if (p->pages->used > packet->pages_alloc) { error_setg(errp, "multifd: received packet " - "with size %d and expected maximum size %d", - p->pages->used, packet->size) ; + "with %d pages and expected maximum pages are %d", + p->pages->used, packet->pages_alloc) ; return -1; } + p->next_packet_size = be32_to_cpu(packet->next_packet_size); p->packet_num = be64_to_cpu(packet->packet_num); if (p->pages->used) { @@ -1073,6 +1102,7 @@ static void *multifd_send_thread(void *opaque) uint64_t packet_num = p->packet_num; uint32_t flags = p->flags; + p->next_packet_size = used * qemu_target_page_size(); multifd_send_fill_packet(p); p->flags = 0; p->num_packets++; @@ -1080,7 +1110,8 @@ static void *multifd_send_thread(void *opaque) p->pages->used = 0; qemu_mutex_unlock(&p->mutex); - trace_multifd_send(p->id, packet_num, used, flags); + trace_multifd_send(p->id, packet_num, used, flags, + p->next_packet_size); ret = qio_channel_write_all(p->c, (void *)p->packet, p->packet_len, &local_err); @@ -1088,9 +1119,12 @@ static void *multifd_send_thread(void *opaque) break; } - ret = qio_channel_writev_all(p->c, p->pages->iov, used, &local_err); - if (ret != 0) { - break; + if (used) { + ret = qio_channel_writev_all(p->c, p->pages->iov, + used, &local_err); + if (ret != 0) { + break; + } } qemu_mutex_lock(&p->mutex); @@ -1148,7 +1182,7 @@ static void multifd_new_send_channel_async(QIOTask *task, gpointer opaque) int multifd_save_setup(void) { int thread_count; - uint32_t page_count = migrate_multifd_page_count(); + uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); uint8_t i; if (!migrate_use_multifd()) { @@ -1312,14 +1346,18 @@ static void *multifd_recv_thread(void *opaque) used = p->pages->used; flags = p->flags; - trace_multifd_recv(p->id, p->packet_num, used, flags); + trace_multifd_recv(p->id, p->packet_num, used, flags, + p->next_packet_size); p->num_packets++; p->num_pages += used; qemu_mutex_unlock(&p->mutex); - ret = qio_channel_readv_all(p->c, p->pages->iov, used, &local_err); - if (ret != 0) { - break; + if (used) { + ret = qio_channel_readv_all(p->c, p->pages->iov, + used, &local_err); + if (ret != 0) { + break; + } } if (flags & MULTIFD_FLAG_SYNC) { @@ -1344,7 +1382,7 @@ static void *multifd_recv_thread(void *opaque) int multifd_load_setup(void) { int thread_count; - uint32_t page_count = migrate_multifd_page_count(); + uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); uint8_t i; if (!migrate_use_multifd()) { diff --git a/migration/rdma.c b/migration/rdma.c index 63c118af09..c1bcece53b 100644 --- a/migration/rdma.c +++ b/migration/rdma.c @@ -641,10 +641,14 @@ static int qemu_rdma_init_one_block(RAMBlock *rb, void *opaque) static int qemu_rdma_init_ram_blocks(RDMAContext *rdma) { RDMALocalBlocks *local = &rdma->local_ram_blocks; + int ret; assert(rdma->blockmap == NULL); memset(local, 0, sizeof *local); - foreach_not_ignored_block(qemu_rdma_init_one_block, rdma); + ret = foreach_not_ignored_block(qemu_rdma_init_one_block, rdma); + if (ret) { + return ret; + } trace_qemu_rdma_init_ram_blocks(local->nb_blocks); rdma->dest_blocks = g_new0(RDMADestBlock, rdma->local_ram_blocks.nb_blocks); diff --git a/migration/tls.c b/migration/tls.c index 3b9e8c9263..5171afc6c4 100644 --- a/migration/tls.c +++ b/migration/tls.c @@ -94,7 +94,7 @@ void migration_tls_channel_process_incoming(MigrationState *s, tioc = qio_channel_tls_new_server( ioc, creds, - NULL, /* XXX pass ACL name */ + s->parameters.tls_authz, errp); if (!tioc) { return; diff --git a/migration/trace-events b/migration/trace-events index 5da4512030..de2e136e57 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -80,13 +80,13 @@ get_queued_page_not_dirty(const char *block_name, uint64_t tmp_offset, unsigned migration_bitmap_sync_start(void) "" migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64 migration_throttle(void) "" -multifd_recv(uint8_t id, uint64_t packet_num, uint32_t used, uint32_t flags) "channel %d packet number %" PRIu64 " pages %d flags 0x%x" +multifd_recv(uint8_t id, uint64_t packet_num, uint32_t used, uint32_t flags, uint32_t next_packet_size) "channel %d packet number %" PRIu64 " pages %d flags 0x%x next packet size %d" multifd_recv_sync_main(long packet_num) "packet num %ld" multifd_recv_sync_main_signal(uint8_t id) "channel %d" multifd_recv_sync_main_wait(uint8_t id) "channel %d" multifd_recv_thread_end(uint8_t id, uint64_t packets, uint64_t pages) "channel %d packets %" PRIu64 " pages %" PRIu64 multifd_recv_thread_start(uint8_t id) "%d" -multifd_send(uint8_t id, uint64_t packet_num, uint32_t used, uint32_t flags) "channel %d packet_num %" PRIu64 " pages %d flags 0x%x" +multifd_send(uint8_t id, uint64_t packet_num, uint32_t used, uint32_t flags, uint32_t next_packet_size) "channel %d packet_num %" PRIu64 " pages %d flags 0x%x next packet size %d" multifd_send_sync_main(long packet_num) "packet num %ld" multifd_send_sync_main_signal(uint8_t id) "channel %d" multifd_send_sync_main_wait(uint8_t id) "channel %d" |