// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package html import ( "io" "os" ) // A parser implements the HTML5 parsing algorithm: // http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#tree-construction type parser struct { // tokenizer provides the tokens for the parser. tokenizer *Tokenizer // tok is the most recently read token. tok Token // Self-closing tags like
are re-interpreted as a two-token sequence: //
followed by . hasSelfClosingToken is true if we have just read // the synthetic start tag and the next one due is the matching end tag. hasSelfClosingToken bool // doc is the document root element. doc *Node // The stack of open elements (section 11.2.3.2) and active formatting // elements (section 11.2.3.3). oe, afe nodeStack // Element pointers (section 11.2.3.4). head, form *Node // Other parsing state flags (section 11.2.3.5). scripting, framesetOK bool // originalIM is the insertion mode to go back to after completing a text // or inTableText insertion mode. originalIM insertionMode // fosterParenting is whether new elements should be inserted according to // the foster parenting rules (section 11.2.5.3). fosterParenting bool } func (p *parser) top() *Node { if n := p.oe.top(); n != nil { return n } return p.doc } // stopTags for use in popUntil. These come from section 11.2.3.2. var ( defaultScopeStopTags = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object"} listItemScopeStopTags = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object", "ol", "ul"} buttonScopeStopTags = []string{"applet", "caption", "html", "table", "td", "th", "marquee", "object", "button"} tableScopeStopTags = []string{"html", "table"} ) // stopTags for use in clearStackToContext. var ( tableRowContextStopTags = []string{"tr", "html"} ) // popUntil pops the stack of open elements at the highest element whose tag // is in matchTags, provided there is no higher element in stopTags. It returns // whether or not there was such an element. If there was not, popUntil leaves // the stack unchanged. // // For example, if the stack was: // ["html", "body", "font", "table", "b", "i", "u"] // then popUntil([]string{"html, "table"}, "font") would return false, but // popUntil([]string{"html, "table"}, "i") would return true and the resultant // stack would be: // ["html", "body", "font", "table", "b"] // // If an element's tag is in both stopTags and matchTags, then the stack will // be popped and the function returns true (provided, of course, there was no // higher element in the stack that was also in stopTags). For example, // popUntil([]string{"html, "table"}, "table") would return true and leave: // ["html", "body", "font"] func (p *parser) popUntil(stopTags []string, matchTags ...string) bool { if i := p.indexOfElementInScope(stopTags, matchTags...); i != -1 { p.oe = p.oe[:i] return true } return false } // indexOfElementInScope returns the index in p.oe of the highest element // whose tag is in matchTags that is in scope according to stopTags. // If no matching element is in scope, it returns -1. func (p *parser) indexOfElementInScope(stopTags []string, matchTags ...string) int { for i := len(p.oe) - 1; i >= 0; i-- { tag := p.oe[i].Data for _, t := range matchTags { if t == tag { return i } } for _, t := range stopTags { if t == tag { return -1 } } } return -1 } // elementInScope is like popUntil, except that it doesn't modify the stack of // open elements. func (p *parser) elementInScope(stopTags []string, matchTags ...string) bool { return p.indexOfElementInScope(stopTags, matchTags...) != -1 } // addChild adds a child node n to the top element, and pushes n onto the stack // of open elements if it is an element node. func (p *parser) addChild(n *Node) { if p.fosterParenting { p.fosterParent(n) } else { p.top().Add(n) } if n.Type == ElementNode { p.oe = append(p.oe, n) } } // fosterParent adds a child node according to the foster parenting rules. // Section 11.2.5.3, "foster parenting". func (p *parser) fosterParent(n *Node) { var table, parent *Node var i int for i = len(p.oe) - 1; i >= 0; i-- { if p.oe[i].Data == "table" { table = p.oe[i] break } } if table == nil { // The foster parent is the html element. parent = p.oe[0] } else { parent = table.Parent } if parent == nil { parent = p.oe[i-1] } var child *Node for i, child = range parent.Child { if child == table { break } } if i > 0 && parent.Child[i-1].Type == TextNode && n.Type == TextNode { parent.Child[i-1].Data += n.Data return } if i == len(parent.Child) { parent.Add(n) } else { // Insert n into parent.Child at index i. parent.Child = append(parent.Child[:i+1], parent.Child[i:]...) parent.Child[i] = n n.Parent = parent } } // addText adds text to the preceding node if it is a text node, or else it // calls addChild with a new text node. func (p *parser) addText(text string) { // TODO: distinguish whitespace text from others. t := p.top() if i := len(t.Child); i > 0 && t.Child[i-1].Type == TextNode { t.Child[i-1].Data += text return } p.addChild(&Node{ Type: TextNode, Data: text, }) } // addElement calls addChild with an element node. func (p *parser) addElement(tag string, attr []Attribute) { p.addChild(&Node{ Type: ElementNode, Data: tag, Attr: attr, }) } // Section 11.2.3.3. func (p *parser) addFormattingElement(tag string, attr []Attribute) { p.addElement(tag, attr) p.afe = append(p.afe, p.top()) // TODO. } // Section 11.2.3.3. func (p *parser) clearActiveFormattingElements() { for { n := p.afe.pop() if len(p.afe) == 0 || n.Type == scopeMarkerNode { return } } } // Section 11.2.3.3. func (p *parser) reconstructActiveFormattingElements() { n := p.afe.top() if n == nil { return } if n.Type == scopeMarkerNode || p.oe.index(n) != -1 { return } i := len(p.afe) - 1 for n.Type != scopeMarkerNode && p.oe.index(n) == -1 { if i == 0 { i = -1 break } i-- n = p.afe[i] } for { i++ clone := p.afe[i].clone() p.addChild(clone) p.afe[i] = clone if i == len(p.afe)-1 { break } } } // read reads the next token. This is usually from the tokenizer, but it may // be the synthesized end tag implied by a self-closing tag. func (p *parser) read() os.Error { if p.hasSelfClosingToken { p.hasSelfClosingToken = false p.tok.Type = EndTagToken p.tok.Attr = nil return nil } p.tokenizer.Next() p.tok = p.tokenizer.Token() switch p.tok.Type { case ErrorToken: return p.tokenizer.Error() case SelfClosingTagToken: p.hasSelfClosingToken = true p.tok.Type = StartTagToken } return nil } // Section 11.2.4. func (p *parser) acknowledgeSelfClosingTag() { p.hasSelfClosingToken = false } // An insertion mode (section 11.2.3.1) is the state transition function from // a particular state in the HTML5 parser's state machine. It updates the // parser's fields depending on parser.token (where ErrorToken means EOF). In // addition to returning the next insertionMode state, it also returns whether // the token was consumed. type insertionMode func(*parser) (insertionMode, bool) // useTheRulesFor runs the delegate insertionMode over p, returning the actual // insertionMode unless the delegate caused a state transition. // Section 11.2.3.1, "using the rules for". func useTheRulesFor(p *parser, actual, delegate insertionMode) (insertionMode, bool) { im, consumed := delegate(p) // TODO: do we need to update p.originalMode if it equals delegate? if im != delegate { return im, consumed } return actual, consumed } // setOriginalIM sets the insertion mode to return to after completing a text or // inTableText insertion mode. // Section 11.2.3.1, "using the rules for". func (p *parser) setOriginalIM(im insertionMode) { if p.originalIM != nil { panic("html: bad parser state: originalIM was set twice") } p.originalIM = im } // Section 11.2.3.1, "reset the insertion mode". func (p *parser) resetInsertionMode() insertionMode { for i := len(p.oe) - 1; i >= 0; i-- { n := p.oe[i] if i == 0 { // TODO: set n to the context element, for HTML fragment parsing. } switch n.Data { case "select": return inSelectIM case "td", "th": return inCellIM case "tr": return inRowIM case "tbody", "thead", "tfoot": return inTableBodyIM case "caption": // TODO: return inCaptionIM case "colgroup": // TODO: return inColumnGroupIM case "table": return inTableIM case "head": return inBodyIM case "body": return inBodyIM case "frameset": // TODO: return inFramesetIM case "html": return beforeHeadIM } } return inBodyIM } // Section 11.2.5.4.1. func initialIM(p *parser) (insertionMode, bool) { switch p.tok.Type { case CommentToken: p.doc.Add(&Node{ Type: CommentNode, Data: p.tok.Data, }) return initialIM, true case DoctypeToken: p.doc.Add(&Node{ Type: DoctypeNode, Data: p.tok.Data, }) return beforeHTMLIM, true } // TODO: set "quirks mode"? It's defined in the DOM spec instead of HTML5 proper, // and so switching on "quirks mode" might belong in a different package. return beforeHTMLIM, false } // Section 11.2.5.4.2. func beforeHTMLIM(p *parser) (insertionMode, bool) { var ( add bool attr []Attribute implied bool ) switch p.tok.Type { case ErrorToken: implied = true case TextToken: // TODO: distinguish whitespace text from others. implied = true case StartTagToken: if p.tok.Data == "html" { add = true attr = p.tok.Attr } else { implied = true } case EndTagToken: switch p.tok.Data { case "head", "body", "html", "br": implied = true default: // Ignore the token. } case CommentToken: p.doc.Add(&Node{ Type: CommentNode, Data: p.tok.Data, }) return beforeHTMLIM, true } if add || implied { p.addElement("html", attr) } return beforeHeadIM, !implied } // Section 11.2.5.4.3. func beforeHeadIM(p *parser) (insertionMode, bool) { var ( add bool attr []Attribute implied bool ) switch p.tok.Type { case ErrorToken: implied = true case TextToken: // TODO: distinguish whitespace text from others. implied = true case StartTagToken: switch p.tok.Data { case "head": add = true attr = p.tok.Attr case "html": return useTheRulesFor(p, beforeHeadIM, inBodyIM) default: implied = true } case EndTagToken: switch p.tok.Data { case "head", "body", "html", "br": implied = true default: // Ignore the token. } case CommentToken: p.addChild(&Node{ Type: CommentNode, Data: p.tok.Data, }) return beforeHeadIM, true } if add || implied { p.addElement("head", attr) } return inHeadIM, !implied } // Section 11.2.5.4.4. func inHeadIM(p *parser) (insertionMode, bool) { var ( pop bool implied bool ) switch p.tok.Type { case ErrorToken, TextToken: implied = true case StartTagToken: switch p.tok.Data { case "meta": // TODO. case "script", "title": p.addElement(p.tok.Data, p.tok.Attr) p.setOriginalIM(inHeadIM) return textIM, true default: implied = true } case EndTagToken: if p.tok.Data == "head" { pop = true } // TODO. case CommentToken: p.addChild(&Node{ Type: CommentNode, Data: p.tok.Data, }) return inHeadIM, true } if pop || implied { n := p.oe.pop() if n.Data != "head" { panic("html: bad parser state: element not found, in the in-head insertion mode") } return afterHeadIM, !implied } return inHeadIM, !implied } // Section 11.2.5.4.6. func afterHeadIM(p *parser) (insertionMode, bool) { var ( add bool attr []Attribute framesetOK bool implied bool ) switch p.tok.Type { case ErrorToken, TextToken: implied = true framesetOK = true case StartTagToken: switch p.tok.Data { case "html": // TODO. case "body": add = true attr = p.tok.Attr framesetOK = false case "frameset": // TODO. case "base", "basefont", "bgsound", "link", "meta", "noframes", "script", "style", "title": // TODO. case "head": // TODO. default: implied = true framesetOK = true } case EndTagToken: // TODO. case CommentToken: p.addChild(&Node{ Type: CommentNode, Data: p.tok.Data, }) return afterHeadIM, true } if add || implied { p.addElement("body", attr) p.framesetOK = framesetOK } return inBodyIM, !implied } // Section 11.2.5.4.7. func inBodyIM(p *parser) (insertionMode, bool) { switch p.tok.Type { case TextToken: p.reconstructActiveFormattingElements() p.addText(p.tok.Data) p.framesetOK = false case StartTagToken: switch p.tok.Data { case "address", "article", "aside", "blockquote", "center", "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", "footer", "header", "hgroup", "menu", "nav", "ol", "p", "section", "summary", "ul": p.popUntil(buttonScopeStopTags, "p") p.addElement(p.tok.Data, p.tok.Attr) case "h1", "h2", "h3", "h4", "h5", "h6": p.popUntil(buttonScopeStopTags, "p") switch n := p.top(); n.Data { case "h1", "h2", "h3", "h4", "h5", "h6": p.oe.pop() } p.addElement(p.tok.Data, p.tok.Attr) case "a": if n := p.afe.forTag("a"); n != nil { p.inBodyEndTagFormatting("a") p.oe.remove(n) p.afe.remove(n) } p.reconstructActiveFormattingElements() p.addFormattingElement(p.tok.Data, p.tok.Attr) case "b", "big", "code", "em", "font", "i", "s", "small", "strike", "strong", "tt", "u": p.reconstructActiveFormattingElements() p.addFormattingElement(p.tok.Data, p.tok.Attr) case "applet", "marquee", "object": p.reconstructActiveFormattingElements() p.addElement(p.tok.Data, p.tok.Attr) p.afe = append(p.afe, &scopeMarker) p.framesetOK = false case "area", "br", "embed", "img", "input", "keygen", "wbr": p.reconstructActiveFormattingElements() p.addElement(p.tok.Data, p.tok.Attr) p.oe.pop() p.acknowledgeSelfClosingTag() p.framesetOK = false case "table": p.popUntil(buttonScopeStopTags, "p") // TODO: skip this step in quirks mode. p.addElement(p.tok.Data, p.tok.Attr) p.framesetOK = false return inTableIM, true case "hr": p.popUntil(buttonScopeStopTags, "p") p.addElement(p.tok.Data, p.tok.Attr) p.oe.pop() p.acknowledgeSelfClosingTag() p.framesetOK = false case "select": p.reconstructActiveFormattingElements() p.addElement(p.tok.Data, p.tok.Attr) p.framesetOK = false // TODO: detect