From 8c70d39212801e9a9d8c396574556548751b90ea Mon Sep 17 00:00:00 2001
From: Ben Miller <bjmllr@gmail.com>
Date: Mon, 20 Apr 2015 07:21:33 -0400
Subject: [PATCH] parse.y: add heredoc <<~ syntax (Feature #9098)

---
 doc/syntax/literals.rdoc   |  19 +++++
 parse.y                    | 193 +++++++++++++++++++++++++++++++++++++++++++--
 test/ripper/test_ripper.rb |  33 ++++++++
 test/ruby/test_syntax.rb   | 101 ++++++++++++++++++++++++
 4 files changed, 338 insertions(+), 8 deletions(-)

diff --git a/doc/syntax/literals.rdoc b/doc/syntax/literals.rdoc
index cbd18e9..66e5ffc 100644
--- a/doc/syntax/literals.rdoc
+++ b/doc/syntax/literals.rdoc
@@ -196,6 +196,25 @@ Note that the while the closing identifier may be indented, the content is
 always treated as if it is flush left.  If you indent the content those spaces
 will appear in the output.
 
+To have indented content as well as an indented closing identifier, you can use
+a "squiggly" heredoc, which uses a "~" instead of a "-" after <tt><<</tt>:
+
+    expected_result = <<~SQUIGGLY_HEREDOC
+      This would contain specially formatted text.
+
+      That might span many lines
+    SQUIGGLY_HEREDOC
+
+The indentation of the least-indented line will be removed from each line of
+the content.  Note that empty lines and lines consisting solely of literal tabs
+and spaces will be ignored for the purposes of determining indentation, but
+escaped tabs and spaces are considered non-indentation characters.
+
+If both tabs and spaces are used as indentation in the same heredoc,
+tabs are considered as equal to 8 spaces.  If the indentation of the
+least-indented line falls in the middle of a leading tab, only
+indentation to the left of that tab will be removed.
+
 A heredoc allows interpolation and escaped characters.  You may disable
 interpolation and escaping by surrounding the opening identifier with single
 quotes:
diff --git a/parse.y b/parse.y
index 3a51893..3a9c9a9 100644
--- a/parse.y
+++ b/parse.y
@@ -255,6 +255,8 @@ struct parser_params {
     int toksiz;
     int tokline;
     int heredoc_end;
+    int heredoc_indent;
+    int heredoc_line_indent;
     char *tokenbuf;
     NODE *deferred_nodes;
     struct local_vars *lvtbl;
@@ -345,6 +347,8 @@ static int parser_yyerror(struct parser_params*, const char*);
 #define lex_p			(parser->lex.pcur)
 #define lex_pend		(parser->lex.pend)
 #define heredoc_end		(parser->heredoc_end)
+#define heredoc_indent		(parser->heredoc_indent)
+#define heredoc_line_indent	(parser->heredoc_line_indent)
 #define command_start		(parser->command_start)
 #define deferred_nodes		(parser->deferred_nodes)
 #define lex_gets_ptr		(parser->lex.gets_ptr)
@@ -485,6 +489,10 @@ static int reg_fragment_check_gen(struct parser_params*, VALUE, int);
 static NODE *reg_named_capture_assign_gen(struct parser_params* parser, VALUE regexp, NODE *match);
 #define reg_named_capture_assign(regexp,match) reg_named_capture_assign_gen(parser,(regexp),(match))
 
+static NODE *parser_heredoc_dedent(struct parser_params*,NODE*);
+# define heredoc_dedent(str) parser_heredoc_dedent(parser, (str))
+
+
 #define get_id(id) (id)
 #define get_value(val) (val)
 #else
@@ -668,6 +676,9 @@ new_args_tail_gen(struct parser_params *parser, VALUE k, VALUE kr, VALUE b)
 
 #define new_defined(expr) dispatch1(defined, (expr))
 
+static VALUE parser_heredoc_dedent_ripper(struct parser_params*,VALUE);
+# define heredoc_dedent_ripper(str) parser_heredoc_dedent_ripper(parser, (str))
+
 #define FIXME 0
 
 #endif /* RIPPER */
@@ -3885,6 +3896,7 @@ strings		: string
 			else {
 			    node = evstr2dstr(node);
 			}
+			heredoc_indent = 0;
 			$$ = node;
 		    /*%
 			$$ = $1;
@@ -3907,9 +3919,10 @@ string		: tCHAR
 string1		: tSTRING_BEG string_contents tSTRING_END
 		    {
 		    /*%%%*/
-			$$ = $2;
+			$$ = heredoc_dedent($2);
 		    /*%
-			$$ = dispatch1(string_literal, $2);
+			$$ = dispatch1(string_literal, 
+                                       heredoc_dedent_ripper($2));
 		    %*/
 		    }
 		;
@@ -3934,9 +3947,10 @@ xstring		: tXSTRING_BEG xstring_contents tSTRING_END
 				break;
 			    }
 			}
-			$$ = node;
+			$$ = heredoc_dedent(node);
 		    /*%
-			$$ = dispatch1(xstring_literal, $2);
+			$$ = dispatch1(xstring_literal, 
+				       heredoc_dedent_ripper($2));
 		    %*/
 		    }
 		;
@@ -4317,6 +4331,10 @@ string_content	: tSTRING_CONTENT
 			$<num>$ = brace_nest;
 			brace_nest = 0;
 		    }
+		    {
+			$<num>$ = heredoc_indent;
+			heredoc_indent = 0;
+		    }
 		  compstmt tSTRING_DEND
 		    {
 			cond_stack = $<val>1;
@@ -4324,11 +4342,12 @@ string_content	: tSTRING_CONTENT
 			lex_strterm = $<node>3;
 			lex_state = $<num>4;
 			brace_nest = $<num>5;
+			heredoc_indent = $<num>6;
 		    /*%%%*/
-			if ($6) $6->flags &= ~NODE_FL_NEWLINE;
-			$$ = new_evstr($6);
+			if ($7) $7->flags &= ~NODE_FL_NEWLINE;
+			$$ = new_evstr($7);
 		    /*%
-			$$ = dispatch1(string_embexpr, $6);
+			$$ = dispatch1(string_embexpr, $7);
 		    %*/
 		    }
 		;
@@ -5693,6 +5712,7 @@ rb_parser_compile_file_path(VALUE vparser, VALUE fname, VALUE file, int start)
 #define STR_FUNC_SYMBOL 0x10
 #define STR_FUNC_INDENT 0x20
 #define STR_FUNC_LABEL  0x40
+#define STR_FUNC_DEDENT 0x80
 
 enum string_type {
     str_label  = STR_FUNC_LABEL,
@@ -6211,6 +6231,23 @@ parser_tokadd_string(struct parser_params *parser,
     } while (0)
 
     while ((c = nextc()) != -1) {
+	if (heredoc_indent > 0) {
+	    if (heredoc_line_indent == -1) {
+		if (c == '\n') heredoc_line_indent = 0;
+	    } else {
+		if (c == ' ') {
+		    heredoc_line_indent++;
+		} else if (c == '\t') {
+		    heredoc_line_indent += 8;
+		} else if (c != '\n') {
+		    if (heredoc_indent > heredoc_line_indent) {
+			heredoc_indent = heredoc_line_indent;
+		    }
+		    heredoc_line_indent = -1;
+		}
+	    }
+	}
+
 	if (paren && c == paren) {
 	    ++*nest;
 	}
@@ -6471,6 +6508,11 @@ parser_heredoc_identifier(struct parser_params *parser)
     if (c == '-') {
 	c = nextc();
 	func = STR_FUNC_INDENT;
+    } else if (c == '~') {
+	c = nextc();
+	func = STR_FUNC_INDENT;
+	heredoc_indent = INT_MAX;
+	heredoc_line_indent = 0;
     }
     switch (c) {
       case '\'':
@@ -6495,7 +6537,9 @@ parser_heredoc_identifier(struct parser_params *parser)
       default:
 	if (!parser_is_identchar()) {
 	    pushback(c);
-	    if (func & STR_FUNC_INDENT) {
+	    if (heredoc_indent > 0) {
+		pushback('~');
+	    } else if (func & STR_FUNC_INDENT) {
 		pushback('-');
 	    }
 	    return 0;
@@ -6541,6 +6585,139 @@ parser_heredoc_restore(struct parser_params *parser, NODE *here)
     ripper_flush(parser);
 }
 
+static VALUE
+parser_heredoc_dedent_string(struct parser_params *parser, VALUE input,
+			     long *count_indent, long *copy_indent)
+{
+    long len, out_len;
+    char *str, *p, *out_p, *end;
+    VALUE output;
+    
+    len = RSTRING_LEN(input);
+    out_len = 0;
+    str = RSTRING_PTR(input);
+    end = &str[len];
+
+    p = str;
+    while (p < end) {
+	while (p < end && *count_indent > 0) {
+	    if (*p == ' ') {
+		p++;
+		(*count_indent)--;
+	    } else if (*p == '\t' && *count_indent >= 8) {
+		p++;
+		*count_indent -= 8;
+	    } else if (*p == '\t' && heredoc_indent % 8) {
+		/* Inconsistent indentation requires us to back up to the
+		   previous tab stop */
+		heredoc_indent = heredoc_indent - (heredoc_indent % 8);
+		*count_indent = *copy_indent = heredoc_indent;
+		return parser_heredoc_dedent_string(parser, input,
+						    count_indent, copy_indent);
+	    } else {
+		break;
+	    }
+	}
+
+	for (; p < end && *p != '\n'; p++) out_len++;
+	if (p < end && *p == '\n') {
+	    *count_indent = heredoc_indent;
+	    out_len++;
+	    p++;
+	}
+    }
+
+    output = rb_str_new(0, out_len);
+    out_p = RSTRING_PTR(output);
+
+    p = str;
+    while (p < end) {
+	while (p < end && *copy_indent > 0) {
+	    if (*p == ' ') {
+		p++;
+		(*copy_indent)--;
+	    } else if (*p == '\t' && *copy_indent >= 8) {
+		p++;
+		*copy_indent -= 8;
+	    } else {
+		break;
+	    }
+	}
+
+	while (p < end && *p != '\n') *out_p++ = *p++;
+	if (p < end && *p == '\n') {
+	    *copy_indent = heredoc_indent;
+	    *out_p++ = *p++;
+	}
+    }
+
+    return output;
+}
+
+#ifndef RIPPER
+static NODE *
+parser_heredoc_dedent(struct parser_params *parser, NODE *root)
+{
+    long count_indent, copy_indent;
+    VALUE output;
+    NODE *node, *str_node;
+
+    if (heredoc_indent <= 0) return root;
+
+    node = str_node = root;
+    count_indent = copy_indent = heredoc_indent;
+
+    while (str_node) {
+	output = parser_heredoc_dedent_string(parser, str_node->nd_lit,
+	    &count_indent, &copy_indent);
+
+	dispose_string(str_node->nd_lit);
+	str_node->nd_lit = output;
+
+	str_node = 0;
+	while (node = node->nd_next) {
+	    if (nd_type(node) != NODE_ARRAY) break;
+	    if (nd_type(node->nd_head) == NODE_STR ||
+		nd_type(node->nd_head) == NODE_DSTR) {
+		str_node = node->nd_head;
+		break;
+	    }
+	}
+    }
+
+    return root;
+}
+#else /* RIPPER */
+static VALUE
+parser_heredoc_dedent_ripper(struct parser_params *parser, VALUE array)
+{
+    long count_indent, copy_indent, array_len, i;
+    VALUE e, sym, ret;
+
+    if (heredoc_indent <= 0) return array;
+
+    count_indent = copy_indent = heredoc_indent;
+
+    array_len = RARRAY_LEN(array);
+    for (i = 0; i < array_len; i++) {
+	e = rb_ary_entry(array, i);
+	if (TYPE(e) == T_ARRAY && TYPE(sym = rb_ary_entry(e, 0)) == T_SYMBOL) {
+	    if (rb_to_id(sym) != rb_intern("string_content") &&
+	    	rb_to_id(sym) != rb_intern("@tstring_content")) continue;
+	    ret = parser_heredoc_dedent_string(parser, rb_ary_entry(e, 1),
+					       &count_indent, &copy_indent);
+	    rb_ary_store(e, 1, ret);
+	} else if (TYPE(e) == T_STRING) {
+	    ret = parser_heredoc_dedent_string(parser, e,
+					       &count_indent, &copy_indent);
+	    rb_ary_store(array, i, ret);
+	}
+    }
+
+    return array;
+}
+#endif
+
 static int
 parser_whole_match_p(struct parser_params *parser,
     const char *eos, long len, int indent)
diff --git a/test/ripper/test_ripper.rb b/test/ripper/test_ripper.rb
index 1544e56..f2ea441 100644
--- a/test/ripper/test_ripper.rb
+++ b/test/ripper/test_ripper.rb
@@ -60,4 +60,37 @@ def test_yydebug_equals
     assert_predicate @ripper, :yydebug
   end
 
+  def test_squiggly_heredoc
+    assert_equal(Ripper.sexp(<<-eos), Ripper.sexp(<<-eos))
+    <<-eot
+asdf
+    eot
+    eos
+    <<~eot
+      asdf
+    eot
+    eos
+  end
+
+  def test_squiggly_heredoc_with_interpolated_expression
+    sexp1 = Ripper.sexp(<<-eos)
+<<-eot
+a\#{1}z
+eot
+    eos
+
+    sexp2 = Ripper.sexp(<<-eos)
+<<~eot
+  a\#{1}z
+eot
+    eos
+
+    pos = lambda do |s|
+      s.fetch(1).fetch(0).fetch(1).fetch(2).fetch(1).fetch(0).fetch(2)
+    end
+    assert_not_equal pos[sexp1], pos[sexp2]
+    pos[sexp1].clear
+    pos[sexp2].clear
+    assert_equal sexp1, sexp2
+  end
 end if ripper_test
diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb
index e2cd389..dab47da 100644
--- a/test/ruby/test_syntax.rb
+++ b/test/ruby/test_syntax.rb
@@ -475,6 +475,107 @@ def test_lineno_command_call_quote
     assert_equal(expected, actual, "#{Bug7559}: ")
   end
 
+  def test_dedented_heredoc_without_indentation
+    assert_equal(" y\nz\n", <<~eos)
+ y
+z
+    eos
+  end
+
+  def test_dedented_heredoc_with_indentation
+    assert_equal(" a\nb\n", <<~eos)
+     a
+    b
+    eos
+  end
+
+  def test_dedented_heredoc_with_blank_less_indented_line
+    # the blank line has two leading spaces
+    assert_equal("a\n\nb\n", <<~eos)
+    a
+  
+    b
+    eos
+  end
+
+  def test_dedented_heredoc_with_blank_less_indented_line_escaped
+    assert_equal("    a\n  \n    b\n", <<~eos)
+    a
+\ \ 
+    b
+    eos
+  end
+
+  def test_dedented_heredoc_with_blank_more_indented_line
+    # the blank line has six leading spaces
+    assert_equal("a\n  \nb\n", <<~eos)
+    a
+      
+    b
+    eos
+  end
+
+  def test_dedented_heredoc_with_blank_more_indented_line_escaped
+    assert_equal("    a\n      \n    b\n", <<~eos)
+    a
+\ \ \ \ \ \ 
+    b
+    eos
+  end
+
+  def test_dedented_heredoc_with_empty_line
+    assert_equal(<<-eos, <<~eos)
+This would contain specially formatted text.
+
+That might span many lines
+    eos
+      This would contain specially formatted text.
+
+      That might span many lines
+    eos
+  end
+
+  def test_dedented_heredoc_with_interpolated_expression
+      assert_equal(<<-eos, <<~eos)
+ #{1}a
+zy
+      eos
+  #{1}a
+ zy
+      eos
+  end
+
+  def test_dedented_heredoc_with_interpolated_string
+    w = ""
+    assert_equal(<<-eos, <<~eos)
+#{w} a
+ zy
+    eos
+ #{w} a
+  zy
+    eos
+  end
+
+  def test_dedented_heredoc_mixing_tab_with_space
+    assert_equal(<<-eos, <<~eos)
+16 spaces
+2 tabs
+    eos
+                16 spaces
+		2 tabs
+    eos
+  end
+
+  def test_dedented_heredoc_with_inconsistent_indentation_preserves_tab
+    assert_equal(<<-eos, <<~eos)
+	2 tabs
+  10 spaces
+    eos
+		2 tabs
+          10 spaces
+    eos
+  end
+
   def test_lineno_after_heredoc
     bug7559 = '[ruby-dev:46737]'
     expected, _, actual = __LINE__, <<eom, __LINE__
-- 
2.6.2

