11package lib
22
3+ import java .time .format .{DateTimeFormatter , DateTimeFormatterBuilder }
4+ import java .time .temporal .ChronoField ._
5+ import java .time .{LocalDateTime , ZoneId , ZonedDateTime }
6+ import javax .mail .internet .MailDateFormat
7+
8+ import com .madgag .okhttpscala ._
9+ import com .squareup .okhttp
10+ import com .squareup .okhttp .OkHttpClient
11+ import controllers .Application ._
12+ import fastparse .core .Result
13+ import lib .Email .Addresses
14+ import lib .model .PatchParsing
15+ import org .jsoup .Jsoup
16+ import org .jsoup .nodes .Element
17+ import play .api .libs .json .Json
18+
19+ import scala .collection .convert .wrapAsScala ._
20+ import scala .concurrent .{ExecutionContext , Future }
21+
22+ case class MessageSummary (id : String , subject : String , date : ZonedDateTime , addresses : Addresses , groupLink : String )
23+
24+ object MessageSummary {
25+ implicit val formatsAddresses = Json .format[Addresses ]
26+ implicit val formatsMessageSummary = Json .format[MessageSummary ]
27+
28+ def fromRawMessage (rawMessage : String , articleUrl : String ): MessageSummary = {
29+ val Result .Success (headers, _) = PatchParsing .headers.parse(rawMessage)
30+ val headerMap = headers.toMap
31+
32+ val messageId = headerMap(" Message-ID" " <" " >"
33+ val from = headerMap(" From"
34+ val date = new MailDateFormat ().parse(headerMap(" Date" ZoneId .of(" UTC"
35+ MessageSummary (messageId, headerMap(" Subject" Addresses (from), articleUrl)
36+ }
37+ }
38+
39+ object RedirectCapturer {
40+ val okClient = {
41+ val c = new OkHttpClient ()
42+ c.setFollowRedirects(false )
43+ c
44+ }
45+
46+ def redirectFor (url : String )(implicit ec : ExecutionContext ): Future [Option [String ]] = for {
47+ resp <- okClient.execute(new okhttp.Request .Builder ().url(url).build())
48+ } yield {
49+ resp.code match {
50+ case FOUND => Some (resp.header(LOCATION ))
51+ case _ => None
52+ }
53+ }
54+ }
55+
56+
357trait MailArchive {
458 val providerName : String
559
660 val url : String
761
862 def linkFor (messageId : String ): String
63+
64+ def lookupMessage (query : String )(implicit ec : ExecutionContext ): Future [Seq [MessageSummary ]] = Future .successful(Seq .empty)
965}
1066
1167case class Gmane (groupName : String ) extends MailArchive {
@@ -14,6 +70,82 @@ case class Gmane(groupName: String) extends MailArchive {
1470 val url = s " http://dir.gmane.org/gmane. $groupName"
1571
1672 def linkFor (messageId : String ) = s " http://mid.gmane.org/ $messageId"
73+
74+ override def lookupMessage (query : String )(implicit ec : ExecutionContext ) = {
75+ for {
76+ gmaneArticleUrlOpt <- gmaneArticleUrlFor(query)
77+ gmaneRawArticleOpt <- gmaneRawArticleFor(gmaneArticleUrlOpt)
78+ } yield gmaneRawArticleOpt.toSeq
79+ }
80+
81+ def gmaneRawArticleFor (articleUrlOpt : Option [String ])(implicit ec : ExecutionContext ): Future [Option [MessageSummary ]] = {
82+ articleUrlOpt match {
83+ case Some (articleUrl) =>
84+ val okClient = new OkHttpClient ()
85+ for {
86+ resp <- okClient.execute(new okhttp.Request .Builder ().url(articleUrl+ " /raw"
87+ } yield Some (MessageSummary .fromRawMessage(resp.body.string, articleUrl))
88+ case None => Future .successful(None )
89+ }
90+ }
91+
92+ def gmaneArticleUrlFor (messageId : String )(implicit ec : ExecutionContext ): Future [Option [String ]] =
93+ RedirectCapturer .redirectFor(linkFor(messageId))
94+
95+ }
96+
97+ object Marc {
98+ val Git = Marc (" git"
99+
100+ def deobfuscate (emailOrMessageId : String ) = emailOrMessageId.replace(" () " " @" " ! " " ."
101+
102+ /*
103+ * Hacks EVERYWHERE - MARC unfortunately doesn't expose this data in a nice format for us
104+ */
105+ def messageSummaryFor (articleHtml : String ): MessageSummary = {
106+ val elements = Jsoup .parse(articleHtml).select(""" pre b font[size="+1"]"""
107+ val nodes = elements.get(0 ).childNodes().toList
108+ val headerMap = (for { header :: value :: Nil <- nodes.grouped(2 ) } yield {
109+ header.outerHtml.trim.stripSuffix(" :" -> value.asInstanceOf [Element ].html()
110+ }).toMap
111+ val messageId = deobfuscate(headerMap(" Message-ID"
112+ val from = deobfuscate(headerMap(" From"
113+
114+ val ISO_LOCAL_TIME = new DateTimeFormatterBuilder ().appendValue(HOUR_OF_DAY ).appendLiteral(':' ).appendValue(MINUTE_OF_HOUR ).optionalStart.appendLiteral(':' ).appendValue(SECOND_OF_MINUTE ).toFormatter
115+ val ISO_LOCAL_DATE_TIME = new DateTimeFormatterBuilder ().parseCaseInsensitive.append(DateTimeFormatter .ISO_LOCAL_DATE ).appendLiteral('T' ).append(ISO_LOCAL_TIME ).toFormatter
116+
117+ val date = LocalDateTime .parse(headerMap(" Date" ' ' , 'T' ), ISO_LOCAL_DATE_TIME ).atZone(ZoneId .of(" UTC"
118+ MessageSummary (messageId, headerMap(" Subject" Addresses (from), " "
119+ }
120+ }
121+
122+ case class Marc (groupName : String ) extends MailArchive {
123+ val providerName = " MARC"
124+
125+ val url = s " http://marc.info/?l= $groupName"
126+
127+ def linkFor (messageId : String ) = s " http://marc.info/?i= $messageId"
128+
129+ override def lookupMessage (query : String )(implicit ec : ExecutionContext ) = {
130+ for {
131+ articleUrlOpt <- articleUrl(query)
132+ articleMessageSummaryOpt <- messageSummaryFor(articleUrlOpt)
133+ } yield articleMessageSummaryOpt.map(_.copy(id = query)).toSeq
134+ }
135+
136+ def articleUrl (messageId : String )(implicit ec : ExecutionContext ): Future [Option [String ]] =
137+ RedirectCapturer .redirectFor(linkFor(messageId))
138+
139+ def messageSummaryFor (articleUrlOpt : Option [String ])(implicit ec : ExecutionContext ): Future [Option [MessageSummary ]] = {
140+ articleUrlOpt match {
141+ case Some (articleUrl) =>
142+ val okClient = new OkHttpClient ()
143+ for {
144+ resp <- okClient.execute(new okhttp.Request .Builder ().url(articleUrl).build())
145+ } yield Some (Marc .messageSummaryFor(resp.body.string).copy(groupLink = articleUrl))
146+ case None => Future .successful(None )
147+ }
148+ }
17149}
18150
19151object Gmane {
0 commit comments