Serialize length instead of end offset

* See ruby#872.
eregon · Apr 24, 2023 · efd78ef · efd78ef
1 parent 0b97269
commit efd78ef
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 12 deletions.
diff --git a/bin/templates/java/org/yarp/Loader.java.erb b/bin/templates/java/org/yarp/Loader.java.erb
@@ -85,15 +85,17 @@ public class Loader {
     private Nodes.Token loadToken() {
         int type = buffer.get() & 0xFF;
         int startOffset = buffer.getInt();
-        int endOffset = buffer.getInt();
+        int length = buffer.getInt();
+        int endOffset = startOffset + length;
 
         final Nodes.TokenType tokenType = Nodes.TOKEN_TYPES[type];
         return new Nodes.Token(tokenType, startOffset, endOffset);
     }
 
     private Nodes.Location loadLocation() {
         int startOffset = buffer.getInt();
-        int endOffset = buffer.getInt();
+        int length = buffer.getInt();
+        int endOffset = startOffset + length;
         return new Nodes.Location(startOffset, endOffset);
     }
 
@@ -113,7 +115,8 @@ public class Loader {
         int type = buffer.get() & 0xFF;
         int length = buffer.getInt();
         int startOffset = buffer.getInt();
-        int endOffset = buffer.getInt();
+        int length = buffer.getInt();
+        int endOffset = startOffset + length;
 
         switch (type) {
             <%- nodes.each_with_index do |node, index| -%>

diff --git a/bin/templates/lib/yarp/serialize.rb.erb b/bin/templates/lib/yarp/serialize.rb.erb
@@ -62,7 +62,8 @@ module YARP
       end
 
       def load_location
-        start_offset, end_offset = io.read(8).unpack("LL")
+        start_offset, length = io.read(8).unpack("LL")
+        end_offset = start_offset + length
         Location.new(start_offset, end_offset)
       end
 

diff --git a/bin/templates/src/serialize.c.erb b/bin/templates/src/serialize.c.erb
@@ -7,19 +7,21 @@ static void
 serialize_token(yp_parser_t *parser, yp_token_t *token, yp_buffer_t *buffer) {
   assert(token->start);
   assert(token->end);
+  assert(token->start <= token->end);
 
   yp_buffer_append_u8(buffer, token->type);
   yp_buffer_append_u32(buffer, yp_long_to_u32(token->start - parser->start));
-  yp_buffer_append_u32(buffer, yp_long_to_u32(token->end - parser->start));
+  yp_buffer_append_u32(buffer, yp_long_to_u32(token->end - token->start));
 }
 
 static void
 serialize_location(yp_parser_t *parser, yp_location_t *location, yp_buffer_t *buffer) {
   assert(location->start);
   assert(location->end);
+  assert(location->start <= location->end);
 
   yp_buffer_append_u32(buffer, yp_long_to_u32(location->start - parser->start));
-  yp_buffer_append_u32(buffer, yp_long_to_u32(location->end - parser->start));
+  yp_buffer_append_u32(buffer, yp_long_to_u32(location->end - location->start));
 }
 
 void
@@ -29,11 +31,7 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
   size_t offset = buffer->length;
   yp_buffer_append_u32(buffer, 0); /* Updated below */
 
-  assert(node->location.start);
-  assert(node->location.end);
-
-  yp_buffer_append_u32(buffer, yp_long_to_u32(node->location.start - parser->start));
-  yp_buffer_append_u32(buffer, yp_long_to_u32(node->location.end - parser->start));
+  serialize_location(parser, &node->location, buffer);
 
   switch (node->type) {
     <%- nodes.each do |node| -%>

diff --git a/docs/serialization.md b/docs/serialization.md
@@ -18,7 +18,7 @@ After the header comes the body of the serialized string. The body consistents o
 | `1` | node type |
 | `4` | byte offset into the serialized string where this node ends |
 | `4` | byte offset into the source string where this node begins |
-| `4` | byte offset into the source string where this node ends |
+| `4` | length of the node in bytes in the source string |
 
 Each node's child is then appended to the serialized string. The child node types can be determined by referencing `config.yml`. Depending on the type of child node, it could take a couple of different forms, described below: