C websockify: support for binary websocket protocol with HyBi/RFC 6455.

The server prefers binary over base64 encoding, given a choice. This is required as noVNC no longer supports base64 encoding.

other/websocket.c

@@ -271,22 +271,30 @@ int encode_hybi(u_char const *src, size_t srclength,
                 char *target, size_t targsize, unsigned int opcode)
 {
     unsigned long long b64_sz, len_offset = 1, payload_offset = 2, len = 0;
-    
-    if ((int)srclength <= 0)
-    {
-        return 0;
-    }
 
-    b64_sz = ((srclength - 1) / 3) * 4 + 4;
+    if (opcode != OPCODE_TEXT && opcode != OPCODE_BINARY) {
+        handler_emsg("Invalid opcode. Opcode must be 0x01 for text mode, or 0x02 for binary mode.\n");
+        return -1;
+    }
 
     target[0] = (char)(opcode & 0x0F | 0x80);
 
-    if (b64_sz <= 125) {
-        target[1] = (char) b64_sz;
+    if ((int)srclength <= 0) {
+        return 0;
+    }
+
+    if (opcode & OPCODE_TEXT) {
+        len = ((srclength - 1) / 3) * 4 + 4;
+    } else {
+        len = srclength;
+    }
+
+    if (len <= 125) {
+        target[1] = (char) len;
         payload_offset = 2;
-    } else if ((b64_sz > 125) && (b64_sz < 65536)) {
+    } else if ((len > 125) && (len < 65536)) {
         target[1] = (char) 126;
-        *(u_short*)&(target[2]) = htons(b64_sz);
+        *(u_short*)&(target[2]) = htons(len);
         payload_offset = 4;
     } else {
         handler_emsg("Sending frames larger than 65535 bytes not supported\n");
@@ -296,8 +304,13 @@ int encode_hybi(u_char const *src, size_t srclength,
         //payload_offset = 10;
     }
 
-    len = ws_b64_ntop(src, srclength, target+payload_offset, targsize-payload_offset);
-    
+    if (opcode & OPCODE_TEXT) {
+        len = ws_b64_ntop(src, srclength, target+payload_offset, targsize-payload_offset);
+    } else {
+        memcpy(target+payload_offset, src, srclength);
+        len = srclength;
+    }
+
     if (len < 0) {
         return len;
     }
@@ -366,7 +379,7 @@ int decode_hybi(unsigned char *src, size_t srclength,
         //printf("    payload_length: %u, raw remaining: %u\n", payload_length, remaining);
         payload = frame + hdr_length + 4*masked;
 
-        if (*opcode != 1 && *opcode != 2) {
+        if (*opcode != OPCODE_TEXT && *opcode != OPCODE_BINARY) {
             handler_msg("Ignoring non-data frame, opcode 0x%x\n", *opcode);
             continue;
         }
@@ -391,8 +404,13 @@ int decode_hybi(unsigned char *src, size_t srclength,
             payload[i] ^= mask[i%4];
         }
 
-        // base64 decode the data
-        len = ws_b64_pton((const char*)payload, target+target_offset, targsize);
+        if (*opcode & OPCODE_TEXT) {
+            // base64 decode the data
+            len = ws_b64_pton((const char*)payload, target+target_offset, targsize);
+        } else {
+            memcpy(target+target_offset, payload, payload_length);
+            len = payload_length;
+        }
 
         // Restore the first character of the next frame
         payload[payload_length] = save_char;
@@ -573,6 +591,7 @@ ws_ctx_t *do_handshake(int sock) {
     headers_t *headers;
     int len, ret, i, offset;
     ws_ctx_t * ws_ctx;
+    char *response_protocol;
 
     // Peek, but don't read the data
     len = recv(sock, handshake, 1024, MSG_PEEK);
@@ -648,10 +667,23 @@ ws_ctx_t *do_handshake(int sock) {
     }
 
     headers = ws_ctx->headers;
+
+    if (strstr(headers->protocols, "binary")) {
+      ws_ctx->opcode = OPCODE_BINARY;
+      response_protocol = "binary";
+    } else if (strstr(headers->protocols, "base64")) {
+      ws_ctx->opcode = OPCODE_TEXT;
+      response_protocol = "base64";
+    } else {
+      handler_emsg("Invalid protocol '%s', expecting 'binary' or 'base64'\n",
+          headers->protocols);
+      return NULL;
+    }
+
     if (ws_ctx->hybi > 0) {
         handler_msg("using protocol HyBi/IETF 6455 %d\n", ws_ctx->hybi);
         gen_sha1(headers, sha1);
-        sprintf(response, SERVER_HANDSHAKE_HYBI, sha1, "base64");
+        sprintf(response, SERVER_HANDSHAKE_HYBI, sha1, response_protocol);
     } else {
         if (ws_ctx->hixie == 76) {
             handler_msg("using protocol Hixie 76\n");

other/websocket.h

@@ -26,6 +26,9 @@ Sec-WebSocket-Protocol: %s\r\n\
 
 #define POLICY_RESPONSE "<cross-domain-policy><allow-access-from domain=\"*\" to-ports=\"*\" /></cross-domain-policy>\n"
 
+#define OPCODE_TEXT    0x01
+#define OPCODE_BINARY  0x02
+
 typedef struct {
     char path[1024+1];
     char host[1024+1];
@@ -44,6 +47,7 @@ typedef struct {
     SSL       *ssl;
     int        hixie;
     int        hybi;
+    int        opcode;
     headers_t *headers;
     char      *cin_buf;
     char      *cout_buf;

other/websockify.c

@@ -155,7 +155,7 @@ void do_proxy(ws_ctx_t *ws_ctx, int target) {
             cout_start = 0;
             if (ws_ctx->hybi) {
                 cout_end = encode_hybi(ws_ctx->cin_buf, bytes,
-                                   ws_ctx->cout_buf, BUFSIZE, 1);
+                                   ws_ctx->cout_buf, BUFSIZE, ws_ctx->opcode);
             } else {
                 cout_end = encode_hixie(ws_ctx->cin_buf, bytes,
                                     ws_ctx->cout_buf, BUFSIZE);
@@ -202,7 +202,7 @@ void do_proxy(ws_ctx_t *ws_ctx, int target) {
             }
 
             if (opcode == 8) {
-                handler_emsg("client sent orderly close frame\n");
+                handler_msg("client sent orderly close frame\n");
                 break;
             }