From d45fc419fa14cf20ea6c6caf1481853f94a195d0 Mon Sep 17 00:00:00 2001 From: Michael Spang Date: Sun, 1 Nov 2009 14:43:15 -0500 Subject: [PATCH] Switch from SCTP to TCP Turns out SCTP doesn't work inside a container. --- src/Makefile | 2 +- src/ceoc.c | 48 +++++--------- src/daemon.h | 3 - src/dmaster.c | 8 ++- src/dslave.c | 34 +++------- src/net.c | 176 +++++++++++++++----------------------------------- src/net.h | 28 +------- 7 files changed, 85 insertions(+), 214 deletions(-) diff --git a/src/Makefile b/src/Makefile index aa8de8a..d95d48e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -20,7 +20,7 @@ HOME_OBJECTS := homedir.o HOME_LIBS := -lacl HOME_PROGS := op-adduser NET_OBJECTS := net.o gss.o ops.o -NET_LIBS := -lsctp $(shell krb5-config --libs gssapi) +NET_LIBS := $(shell krb5-config --libs gssapi) NET_PROGS := ceod ceoc PROTO_OBJECTS := ceo.pb-c.o PROTO_LIBS := -lprotobuf-c diff --git a/src/ceoc.c b/src/ceoc.c index dc9029d..c939671 100644 --- a/src/ceoc.c +++ b/src/ceoc.c @@ -24,9 +24,8 @@ static void usage() { static void send_gss_token(int sock, struct sockaddr *addr, socklen_t addrlen, gss_buffer_t token) { OM_uint32 maj_stat, min_stat; - if (sctp_sendmsg(sock, token->value, token->length, - addr, addrlen, MSG_AUTH, 0, 0, 0, 0) < 0) - fatalpe("sctp_sendmsg"); + if (ceo_send_message(sock, token->value, token->length, MSG_AUTH)) + fatalpe("write"); maj_stat = gss_release_buffer(&min_stat, token); if (maj_stat != GSS_S_COMPLETE) @@ -35,8 +34,8 @@ static void send_gss_token(int sock, struct sockaddr *addr, socklen_t addrlen, g static void client_gss_auth(int sock, struct sockaddr *addr, socklen_t addrlen) { gss_buffer_desc incoming_tok, outgoing_tok; - struct sctp_meta msg_meta; struct strbuf msg = STRBUF_INIT; + uint32_t msgtype; int complete; complete = initial_client_token(&outgoing_tok); @@ -50,11 +49,11 @@ static void client_gss_auth(int sock, struct sockaddr *addr, socklen_t addrlen) if (complete) break; - if (!receive_one_message(sock, &msg_meta, &msg)) + if (ceo_receive_message(sock, &msg, &msgtype)) fatal("connection closed during auth"); - if (msg_meta.sinfo.sinfo_ppid != MSG_AUTH) - fatal("unexpected message type 0x%x", msg_meta.sinfo.sinfo_ppid); + if (msgtype != MSG_AUTH) + fatal("unexpected message type 0x%x", msgtype); incoming_tok.value = msg.buf; incoming_tok.length = msg.len; @@ -67,9 +66,9 @@ static void client_gss_auth(int sock, struct sockaddr *addr, socklen_t addrlen) void run_remote(struct op *op, struct strbuf *in, struct strbuf *out) { const char *hostname = op->hostname; - int sock = socket(PF_INET, SOCK_SEQPACKET, IPPROTO_SCTP); + int sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); struct sockaddr_in addr; - struct sctp_meta response_meta; + uint32_t msgtype; struct strbuf in_cipher = STRBUF_INIT, out_cipher = STRBUF_INIT; if (!in->len) @@ -80,40 +79,27 @@ void run_remote(struct op *op, struct strbuf *in, struct strbuf *out) { addr.sin_port = htons(9987); addr.sin_addr = op->addr; - struct sctp_event_subscribe events; - memset(&events, 0, sizeof(events)); - events.sctp_data_io_event = 1; - events.sctp_association_event = 1; - events.sctp_address_event = 1; - events.sctp_send_failure_event = 1; - events.sctp_peer_error_event = 1; - events.sctp_shutdown_event = 1; - events.sctp_partial_delivery_event = 1; - events.sctp_adaptation_layer_event = 1; - - if (setsockopt(sock, IPPROTO_SCTP, SCTP_EVENTS, &events, sizeof(events))) - fatalpe("setsockopt"); + if (connect(sock, (struct sockaddr *)&addr, sizeof(addr))) + fatalpe("connect"); client_acquire_creds("ceod", hostname); client_gss_auth(sock, (sa *)&addr, sizeof(addr)); gss_encipher(in, &in_cipher); - if (sctp_sendmsg(sock, in_cipher.buf, in_cipher.len, (struct sockaddr *)&addr, - sizeof(addr), op->id, 0, 0, 0, 0) < 0) - fatalpe("sctp_sendmsg"); + if (ceo_send_message(sock, in_cipher.buf, in_cipher.len, op->id)) + fatalpe("write"); - if (!receive_one_message(sock, &response_meta, &out_cipher)) + if (ceo_receive_message(sock, &out_cipher, &msgtype)) fatal("no response received for op %s", op->name); gss_decipher(&out_cipher, out); - if (response_meta.sinfo.sinfo_ppid != op->id) - fatal("wrong ppid from server: expected %d got %d", op->id, response_meta.sinfo.sinfo_ppid); + if (msgtype != op->id) + fatal("wrong message type from server: expected %d got %d", op->id, msgtype); - if (sctp_sendmsg(sock, NULL, 0, (struct sockaddr *)&addr, - sizeof(addr), 0, SCTP_EOF, 0, 0 ,0) < 0) - fatalpe("sctp_sendmsg"); + if (close(sock)) + fatalpe("close"); strbuf_release(&in_cipher); strbuf_release(&out_cipher); diff --git a/src/daemon.h b/src/daemon.h index 12bf804..146a6ba 100644 --- a/src/daemon.h +++ b/src/daemon.h @@ -5,6 +5,3 @@ extern int fatal_signal; /* dslave.c */ void slave_main(int sock, struct sockaddr *addr); void setup_slave(void); - -/* builtin-adduser.c */ -extern void builtin_adduser(struct sctp_meta *in, struct sctp_meta *out); diff --git a/src/dmaster.c b/src/dmaster.c index 63f0e83..be5c9b5 100644 --- a/src/dmaster.c +++ b/src/dmaster.c @@ -138,7 +138,7 @@ static void accept_one_client(int server) { } static int master_main(void) { - int sock; + int sock, opt; struct sockaddr_in addr; memset(&addr, 0, sizeof(addr)); @@ -146,10 +146,14 @@ static int master_main(void) { addr.sin_port = htons(9987); addr.sin_addr.s_addr = INADDR_ANY; - sock = socket(PF_INET, SOCK_STREAM, IPPROTO_SCTP); + sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); if (sock < 0) fatalpe("socket"); + opt = 1; + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt))) + fatalpe("setsockopt"); + if (bind(sock, (struct sockaddr *)&addr, sizeof(addr))) fatalpe("bind"); diff --git a/src/dslave.c b/src/dslave.c index 8bd511f..1a36dc3 100644 --- a/src/dslave.c +++ b/src/dslave.c @@ -48,20 +48,6 @@ static void setup_slave_sigs(void) { raise(fatal_signal); } -static void setup_client(int client, struct sockaddr *addr) { - struct sctp_event_subscribe events; - memset(&events, 0, sizeof(events)); - events.sctp_data_io_event = 1; - events.sctp_address_event = 1; - events.sctp_send_failure_event = 1; - events.sctp_peer_error_event = 1; - events.sctp_partial_delivery_event = 1; - events.sctp_adaptation_layer_event = 1; - - if (setsockopt(client, IPPROTO_SCTP, SCTP_EVENTS, &events, sizeof(events))) - fatalpe("setsockopt(SCTP_EVENTS)"); -} - static void handle_auth_message(struct strbuf *in, struct strbuf *out) { gss_buffer_desc incoming_tok, outgoing_tok; OM_uint32 maj_stat, min_stat; @@ -113,19 +99,16 @@ static void handle_op_message(uint32_t in_type, struct strbuf *in, struct strbuf strbuf_release(&out_plain); } -static void handle_one_message(int sock, struct sctp_meta *in_meta, struct strbuf *in) { +static void handle_one_message(int sock, struct strbuf *in, uint32_t msgtype) { struct strbuf out = STRBUF_INIT; - uint32_t ppid = in_meta->sinfo.sinfo_ppid; - if (ppid == MSG_AUTH) + if (msgtype == MSG_AUTH) handle_auth_message(in, &out); else - handle_op_message(ppid, in, &out); + handle_op_message(msgtype, in, &out); - if (out.len && sctp_sendmsg(sock, out.buf, out.len, - (sa *)&in_meta->from, in_meta->fromlen, ppid, - 0, 0, 0, 0) < 0) - fatalpe("sctp_sendmsg"); + if (out.len && ceo_send_message(sock, out.buf, out.len, msgtype)) + fatalpe("write"); strbuf_release(&out); } @@ -133,7 +116,7 @@ static void handle_one_message(int sock, struct sctp_meta *in_meta, struct strbu void slave_main(int sock, struct sockaddr *addr) { char addrstr[INET_ADDRSTRLEN]; struct sockaddr_in *addr_in = (struct sockaddr_in *)addr; - struct sctp_meta msg_meta; + uint32_t msgtype; struct strbuf msg = STRBUF_INIT; if (addr->sa_family != AF_INET) @@ -145,12 +128,11 @@ void slave_main(int sock, struct sockaddr *addr) { notice("accepted connection from %s", addrstr); setup_slave_sigs(); - setup_client(sock, addr); while (!terminate) { - if (!receive_one_message(sock, &msg_meta, &msg)) + if (ceo_receive_message(sock, &msg, &msgtype)) break; - handle_one_message(sock, &msg_meta, &msg); + handle_one_message(sock, &msg, msgtype); } notice("connection closed by peer %s", addrstr); diff --git a/src/net.c b/src/net.c index d5db0ad..a238f1f 100644 --- a/src/net.c +++ b/src/net.c @@ -31,139 +31,65 @@ void free_fqdn(void) { strbuf_release(&fqdn); } -static size_t recv_one_message(int sock, struct sctp_meta *msg_meta, struct strbuf *msg, int *notification) { - size_t len = 0; - int flags = 0; - int bytes; +int ceo_send_message(int sock, void *buf, size_t len, uint32_t msgtype) { + uint32_t msgheader[2]; + msgheader[0] = htonl(len); + msgheader[1] = htonl(msgtype); + + if (full_write(sock, msgheader, sizeof(msgheader)) < 0) + fatalpe("write"); + + if (full_write(sock, buf, len) < 0) + fatalpe("write"); + + return 0; +} + +int ceo_receive_message(int sock, struct strbuf *msg, uint32_t *msgtype) { + uint32_t msglen, received = 0; + uint32_t msgheader[2]; + ssize_t bytes; strbuf_reset(msg); - do { - msg_meta->fromlen = sizeof(msg_meta->from); - - bytes = sctp_recvmsg(sock, msg->buf + len, strbuf_avail(msg) - len, - (sa *)&msg_meta->from, &msg_meta->fromlen, &msg_meta->sinfo, &flags); - + while (received < sizeof(msgheader)) { + bytes = read(sock, msgheader, sizeof(msgheader) - received); if (bytes < 0) { if (errno == EAGAIN) continue; - fatalpe("sctp_recvmsg"); + fatalpe("read"); + } + if (!bytes && !received) + return -1; + if (!bytes) + fatalpe("short header received"); + received += bytes; + } + + msglen = ntohl(msgheader[0]); + *msgtype = ntohl(msgheader[1]); + received = 0; + + if (!msglen) + fatal("length is zero in message header"); + + if (msglen > MAX_MSGLEN) + fatal("length is huge in message header"); + + strbuf_grow(msg, msglen); + strbuf_setlen(msg, msglen); + + while (received < msglen) { + bytes = read(sock, msg->buf + received, msglen - received); + if (bytes < 0) { + if (errno == EAGAIN) + continue; + fatalpe("read"); } if (!bytes) - break; - len += bytes; - - if (msg->len > MAX_MSGLEN) - fatal("oversized message received"); - if (strbuf_avail(msg) < MSG_BUFINC) - strbuf_grow(msg, MSG_BUFINC); - - } while (~flags & MSG_EOR); - - if (!bytes && len) - fatalpe("EOF in the middle of a message"); - - *notification = flags & MSG_NOTIFICATION; - if (*notification) { - notification_dbg(msg->buf); - union sctp_notification *sn = (union sctp_notification *) msg->buf; - switch (sn->sn_header.sn_type) { - case SCTP_SHUTDOWN_EVENT: - fatal("connection shut down"); - break; - case SCTP_ASSOC_CHANGE: - switch (sn->sn_assoc_change.sac_state) { - case SCTP_COMM_LOST: - fatal("connection lost"); - break; - case SCTP_SHUTDOWN_COMP: - fatal("shutdown complete"); - break; - case SCTP_CANT_STR_ASSOC: - fatal("connection failed (is ceod running?)"); - break; - } - break; - } + fatal("short message received"); + received += bytes; } - strbuf_setlen(msg, len); - return len; -} - -int receive_one_message(int sock, struct sctp_meta *msg_meta, struct strbuf *msg) { - int notification = 0; - - do { - recv_one_message(sock, msg_meta, msg, ¬ification); - } while (notification); - - return msg->len > 0; -} - -void notification_dbg(char *notification) { - union sctp_notification *sn = (union sctp_notification *) notification; - char *extra; - - switch (sn->sn_header.sn_type) { - case SCTP_ASSOC_CHANGE: - extra = "unknown state"; - switch (sn->sn_assoc_change.sac_state) { - case SCTP_COMM_UP: extra = "established"; break; - case SCTP_COMM_LOST: extra = "lost"; break; - case SCTP_RESTART: extra = "restarted"; break; - case SCTP_SHUTDOWN_COMP: extra = "completed shutdown"; break; - case SCTP_CANT_STR_ASSOC: extra = "cannot start"; break; - } - debug("association changed: association 0x%x %s", sn->sn_assoc_change.sac_assoc_id, extra); - break; - case SCTP_PEER_ADDR_CHANGE: - extra = "unknown state"; - switch (sn->sn_paddr_change.spc_state) { - case SCTP_ADDR_AVAILABLE: extra = "unavailable"; break; - case SCTP_ADDR_UNREACHABLE: extra = "unreachable"; break; - case SCTP_ADDR_REMOVED: extra = "removed"; break; - case SCTP_ADDR_ADDED: extra = "added"; break; - case SCTP_ADDR_MADE_PRIM: extra = "made primary"; break; -#ifdef SCTP_ADDR_CONFIRMED - case SCTP_ADDR_CONFIRMED: extra = "confirmed"; break; -#endif - } - - struct sockaddr_in *sa = (struct sockaddr_in *) &sn->sn_paddr_change.spc_aaddr; - char addr[INET_ADDRSTRLEN]; - inet_ntop(AF_INET, &sa->sin_addr, addr, sizeof(addr)); - debug("peer address change: remote address %s %s", addr, extra); - break; - case SCTP_REMOTE_ERROR: - debug("remote error: association=0x%x error=0x%x", - sn->sn_remote_error.sre_assoc_id, - sn->sn_remote_error.sre_error); - break; - case SCTP_SEND_FAILED: - debug("send failed: association=0x%x error=0x%x", - sn->sn_send_failed.ssf_assoc_id, - sn->sn_send_failed.ssf_error); - break; - case SCTP_ADAPTATION_INDICATION: - debug("adaptation indication: 0x%x", - sn->sn_adaptation_event.sai_adaptation_ind); - break; - case SCTP_PARTIAL_DELIVERY_EVENT: - extra = "unknown indication"; - switch (sn->sn_pdapi_event.pdapi_indication) { - case SCTP_PARTIAL_DELIVERY_ABORTED: - extra = "partial delivery aborted"; - break; - } - debug("partial delivery event: %s", extra); - break; - case SCTP_SHUTDOWN_EVENT: - debug("association 0x%x was shut down", - sn->sn_shutdown_event.sse_assoc_id); - break; - default: - debug("unknown sctp notification type 0x%x\n", - sn->sn_header.sn_type); - } + return 0; } diff --git a/src/net.h b/src/net.h index 8674a82..2015421 100644 --- a/src/net.h +++ b/src/net.h @@ -1,47 +1,23 @@ #include #include #include -#include #include #include -#if ! defined(SCTP_ADDR_CONFIRMED) && defined(__linux__) -#define SCTP_ADDR_CONFIRMED 5 -#endif - -void notification_dbg(char *); - -#ifdef SCTP_ADAPTION_LAYER -#define sctp_adaptation_layer_event sctp_adaption_layer_event -#define sn_adaptation_event sn_adaption_event -#define sai_adaptation_ind sai_adaption_ind -#define SCTP_ADAPTATION_INDICATION SCTP_ADAPTION_INDICATION -#endif - typedef struct sockaddr sa; extern struct strbuf fqdn; extern void setup_fqdn(void); extern void free_fqdn(void); -struct sctp_meta { - struct sockaddr_storage from; - socklen_t fromlen; - struct sctp_sndrcvinfo sinfo; -}; - enum { MSG_AUTH = 0x8000000, MSG_EXPLODE = 0x8000001, }; -#ifdef MSG_ABORT -#define SCTP_ABORT MSG_ABORT -#define SCTP_EOF MSG_EOF -#endif - #define EKERB -2 #define ELDAP -3 #define EHOME -4 -int receive_one_message(int sock, struct sctp_meta *msg_meta, struct strbuf *msg); +int ceo_receive_message(int sock, struct strbuf *msg, uint32_t *msgtype); +int ceo_send_message(int sock, void *msg, size_t len, uint32_t msgtype);