Stable APIs for Go. https://go.code.as

main.go 15KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552
  1. package main
  2. import (
  3. "bytes"
  4. "crypto/tls"
  5. "errors"
  6. "flag"
  7. "fmt"
  8. "io"
  9. "io/ioutil"
  10. "log"
  11. "net/http"
  12. "os"
  13. "regexp"
  14. "strconv"
  15. "strings"
  16. "text/template"
  17. "time"
  18. "golang.org/x/crypto/acme/autocert"
  19. )
  20. var (
  21. httpFlag = flag.String("http", ":8080", "Serve HTTP at given address")
  22. httpsFlag = flag.String("https", "", "Serve HTTPS at given address")
  23. certFlag = flag.String("cert", "", "Use the provided TLS certificate")
  24. keyFlag = flag.String("key", "", "Use the provided TLS key")
  25. acmeFlag = flag.String("acme", "", "Auto-request TLS certs and store in given directory")
  26. domainFlag = flag.String("domain", "gopkg.in", "Domain name")
  27. srcDomainFlag = flag.String("src", "github.com", "Domain name containing source code")
  28. userFlag = flag.String("username", "", "Github username")
  29. )
  30. var httpServer = &http.Server{
  31. ReadTimeout: 30 * time.Second,
  32. WriteTimeout: 5 * time.Minute,
  33. }
  34. var httpClient = &http.Client{
  35. Timeout: 10 * time.Second,
  36. }
  37. var bulkClient = &http.Client{
  38. Timeout: 5 * time.Minute,
  39. }
  40. func main() {
  41. if err := run(); err != nil {
  42. fmt.Fprintf(os.Stderr, "error: %v\n", err)
  43. os.Exit(1)
  44. }
  45. }
  46. func run() error {
  47. flag.Parse()
  48. http.HandleFunc("/", handler)
  49. if *httpFlag == "" && *httpsFlag == "" {
  50. return fmt.Errorf("must provide -http and/or -https")
  51. }
  52. if *acmeFlag != "" && *httpsFlag == "" {
  53. return fmt.Errorf("cannot use -acme without -https")
  54. }
  55. if *acmeFlag != "" && (*certFlag != "" || *keyFlag != "") {
  56. return fmt.Errorf("cannot provide -acme with -key or -cert")
  57. }
  58. if *acmeFlag == "" && (*httpsFlag != "" || *certFlag != "" || *keyFlag != "") && (*httpsFlag == "" || *certFlag == "" || *keyFlag == "") {
  59. return fmt.Errorf("-https -cert and -key must be used together")
  60. }
  61. ch := make(chan error, 2)
  62. if *acmeFlag != "" {
  63. // So a potential error is seen upfront.
  64. if err := os.MkdirAll(*acmeFlag, 0700); err != nil {
  65. return err
  66. }
  67. }
  68. if *httpFlag != "" && (*httpsFlag == "" || *acmeFlag == "") {
  69. server := *httpServer
  70. server.Addr = *httpFlag
  71. go func() {
  72. ch <- server.ListenAndServe()
  73. }()
  74. }
  75. if *httpsFlag != "" {
  76. server := *httpServer
  77. server.Addr = *httpsFlag
  78. if *acmeFlag != "" {
  79. m := autocert.Manager{
  80. ForceRSA: true,
  81. Prompt: autocert.AcceptTOS,
  82. Cache: autocert.DirCache(*acmeFlag),
  83. RenewBefore: 24 * 30 * time.Hour,
  84. HostPolicy: autocert.HostWhitelist(
  85. "localhost",
  86. "gopkg.in",
  87. "p1.gopkg.in",
  88. "p2.gopkg.in",
  89. "p3.gopkg.in",
  90. "mup.labix.org",
  91. ),
  92. Email: "gustavo@niemeyer.net",
  93. }
  94. server.TLSConfig = &tls.Config{
  95. GetCertificate: m.GetCertificate,
  96. }
  97. go func() {
  98. ch <- http.ListenAndServe(":80", m.HTTPHandler(nil))
  99. }()
  100. }
  101. go func() {
  102. ch <- server.ListenAndServeTLS(*certFlag, *keyFlag)
  103. }()
  104. }
  105. return <-ch
  106. }
  107. var gogetTemplate = template.Must(template.New("").Parse(`
  108. <html>
  109. <head>
  110. <meta name="go-import" content="{{.Original.GopkgRoot}} git https://{{.Original.GopkgRoot}}">
  111. {{$root := .GitHubRoot}}{{$tree := .GitHubTree}}<meta name="go-source" content="{{.Original.GopkgRoot}} _ https://{{$root}}/{{.TreeDir}}/{{$tree}}{/dir} https://{{$root}}/{{.BlobDir}}/{{$tree}}{/dir}/{file}#L{line}">
  112. </head>
  113. <body>
  114. go get {{.GopkgPath}}
  115. </body>
  116. </html>
  117. `))
  118. // Repo represents a source code repository on a repo host.
  119. type Repo struct {
  120. User string
  121. Name string
  122. SubPath string
  123. OldFormat bool // The old /v2/pkg format.
  124. MajorVersion Version
  125. // FullVersion is the best version in AllVersions that matches MajorVersion.
  126. // It defaults to InvalidVersion if there are no matches.
  127. FullVersion Version
  128. // AllVersions holds all versions currently available in the repository,
  129. // either coming from branch names or from tag names. Version zero (v0)
  130. // is only present in the list if it really exists in the repository.
  131. AllVersions VersionList
  132. // When there is a redirect in place, these are from the original request.
  133. RedirUser string
  134. RedirName string
  135. }
  136. // SetVersions records in the relevant fields the details about which
  137. // package versions are available in the repository.
  138. func (repo *Repo) SetVersions(all []Version) {
  139. repo.AllVersions = all
  140. for _, v := range repo.AllVersions {
  141. if v.Major == repo.MajorVersion.Major && v.Unstable == repo.MajorVersion.Unstable && repo.FullVersion.Less(v) {
  142. repo.FullVersion = v
  143. }
  144. }
  145. }
  146. // When there is a redirect in place, this will return the original repository
  147. // but preserving the data for the new repository.
  148. func (repo *Repo) Original() *Repo {
  149. if repo.RedirName == "" {
  150. return repo
  151. }
  152. orig := *repo
  153. orig.User = repo.RedirUser
  154. orig.Name = repo.RedirName
  155. return &orig
  156. }
  157. type repoBase struct {
  158. user string
  159. name string
  160. }
  161. var redirect = map[repoBase]repoBase{
  162. // https://github.com/go-fsnotify/fsnotify/issues/1
  163. {"", "fsnotify"}: {"fsnotify", "fsnotify"},
  164. }
  165. // GitHubRoot returns the repository root, without a schema.
  166. func (repo *Repo) GitHubRoot() string {
  167. if *userFlag != "" {
  168. return *srcDomainFlag + "/" + *userFlag + "/" + repo.Name
  169. }
  170. if repo.User == "" {
  171. return *srcDomainFlag + "/" + repo.Name + "/go-" + repo.Name
  172. }
  173. return *srcDomainFlag + "/" + repo.User + "/" + repo.Name
  174. }
  175. // TreeDir returns the repository's web tree dir
  176. func (repo *Repo) TreeDir() string {
  177. if *srcDomainFlag != "github.com" {
  178. return "src/branch"
  179. }
  180. return "tree"
  181. }
  182. // BlobDir returns the repository's dir for source files
  183. func (repo *Repo) BlobDir() string {
  184. if *srcDomainFlag != "github.com" {
  185. return "src/branch"
  186. }
  187. return "blob"
  188. }
  189. // GitHubTree returns the repository tree name for the selected version.
  190. func (repo *Repo) GitHubTree() string {
  191. if repo.FullVersion == InvalidVersion {
  192. return "master"
  193. }
  194. return repo.FullVersion.String()
  195. }
  196. // GopkgRoot returns the package root at gopkg.in, without a schema.
  197. func (repo *Repo) GopkgRoot() string {
  198. return repo.GopkgVersionRoot(repo.MajorVersion)
  199. }
  200. // GopkgPath returns the package path at gopkg.in, without a schema.
  201. func (repo *Repo) GopkgPath() string {
  202. return repo.GopkgVersionRoot(repo.MajorVersion) + repo.SubPath
  203. }
  204. // GopkgVersionRoot returns the package root in gopkg.in for the
  205. // provided version, without a schema.
  206. func (repo *Repo) GopkgVersionRoot(version Version) string {
  207. version.Minor = -1
  208. version.Patch = -1
  209. v := version.String()
  210. if repo.OldFormat {
  211. if repo.User == "" {
  212. return *domainFlag + "/" + v + "/" + repo.Name
  213. } else {
  214. return *domainFlag + "/" + repo.User + "/" + v + "/" + repo.Name
  215. }
  216. } else {
  217. if repo.User == "" {
  218. return *domainFlag + "/" + repo.Name + "." + v
  219. } else {
  220. return *domainFlag + "/" + repo.User + "/" + repo.Name + "." + v
  221. }
  222. }
  223. }
  224. var patternOld = regexp.MustCompile(`^/(?:([a-z0-9][-a-z0-9]+)/)?((?:v0|v[1-9][0-9]*)(?:\.0|\.[1-9][0-9]*){0,2}(?:-unstable)?)/([a-zA-Z][-a-zA-Z0-9]*)(?:\.git)?((?:/[a-zA-Z][-a-zA-Z0-9]*)*)$`)
  225. var patternNew = regexp.MustCompile(`^/(?:([a-zA-Z0-9][-a-zA-Z0-9]+)/)?([a-zA-Z][-.a-zA-Z0-9]*)\.((?:v0|v[1-9][0-9]*)(?:\.0|\.[1-9][0-9]*){0,2}(?:-unstable)?)(?:\.git)?((?:/[a-zA-Z0-9][-.a-zA-Z0-9]*)*)$`)
  226. func handler(resp http.ResponseWriter, req *http.Request) {
  227. if req.URL.Path == "/health-check" {
  228. resp.Write([]byte("ok"))
  229. return
  230. }
  231. log.Printf("%s requested %s", req.RemoteAddr, req.URL)
  232. if req.URL.Path == "/" {
  233. resp.Header().Set("Location", "https://code.as/abunchtell/gopkg")
  234. resp.WriteHeader(http.StatusTemporaryRedirect)
  235. return
  236. }
  237. m := patternNew.FindStringSubmatch(req.URL.Path)
  238. oldFormat := false
  239. if m == nil {
  240. m = patternOld.FindStringSubmatch(req.URL.Path)
  241. if m == nil {
  242. sendNotFound(resp, "Unsupported URL pattern; see the documentation at gopkg.in for details.")
  243. return
  244. }
  245. // "/v2/name" <= "/name.v2"
  246. m[2], m[3] = m[3], m[2]
  247. oldFormat = true
  248. }
  249. if strings.Contains(m[3], ".") {
  250. sendNotFound(resp, "Import paths take the major version only (.%s instead of .%s); see docs at gopkg.in for the reasoning.",
  251. m[3][:strings.Index(m[3], ".")], m[3])
  252. return
  253. }
  254. repo := &Repo{
  255. User: m[1],
  256. Name: m[2],
  257. SubPath: m[4],
  258. OldFormat: oldFormat,
  259. FullVersion: InvalidVersion,
  260. }
  261. if r, ok := redirect[repoBase{repo.User, repo.Name}]; ok {
  262. repo.RedirUser, repo.RedirName = repo.User, repo.Name
  263. repo.User, repo.Name = r.user, r.name
  264. }
  265. var ok bool
  266. repo.MajorVersion, ok = parseVersion(m[3])
  267. if !ok {
  268. sendNotFound(resp, "Version %q improperly considered invalid; please warn the service maintainers.", m[3])
  269. return
  270. }
  271. var changed []byte
  272. var versions VersionList
  273. original, err := fetchRefs(repo)
  274. if err == nil {
  275. changed, versions, err = changeRefs(original, repo.MajorVersion)
  276. repo.SetVersions(versions)
  277. }
  278. switch err {
  279. case nil:
  280. // all ok
  281. case ErrNoRepo:
  282. sendNotFound(resp, "Repository not found at https://%s", repo.GitHubRoot())
  283. return
  284. case ErrNoVersion:
  285. major := repo.MajorVersion
  286. suffix := ""
  287. if major.Unstable {
  288. major.Unstable = false
  289. suffix = unstableSuffix
  290. }
  291. v := major.String()
  292. sendNotFound(resp, `Repository at https://%s has no branch or tag "%s%s", "%s.N%s" or "%s.N.M%s"`, repo.GitHubRoot(), v, suffix, v, suffix, v, suffix)
  293. return
  294. default:
  295. resp.WriteHeader(http.StatusBadGateway)
  296. resp.Write([]byte(fmt.Sprintf("Cannot obtain refs from repo host: %v", err)))
  297. return
  298. }
  299. if repo.SubPath == "/git-upload-pack" {
  300. proxyUploadPack(resp, req, repo)
  301. return
  302. }
  303. if repo.SubPath == "/info/refs" {
  304. resp.Header().Set("Content-Type", "application/x-git-upload-pack-advertisement")
  305. resp.Write(changed)
  306. return
  307. }
  308. resp.Header().Set("Content-Type", "text/html")
  309. if req.FormValue("go-get") == "1" {
  310. // execute simple template when this is a go-get request
  311. err = gogetTemplate.Execute(resp, repo)
  312. if err != nil {
  313. log.Printf("error executing go get template: %s\n", err)
  314. }
  315. return
  316. }
  317. renderPackagePage(resp, req, repo)
  318. }
  319. func sendNotFound(resp http.ResponseWriter, msg string, args ...interface{}) {
  320. if len(args) > 0 {
  321. msg = fmt.Sprintf(msg, args...)
  322. }
  323. resp.WriteHeader(http.StatusNotFound)
  324. resp.Write([]byte(msg))
  325. }
  326. const refsSuffix = ".git/info/refs?service=git-upload-pack"
  327. func proxyUploadPack(resp http.ResponseWriter, req *http.Request, repo *Repo) {
  328. preq, err := http.NewRequest(req.Method, "https://"+repo.GitHubRoot()+"/git-upload-pack", req.Body)
  329. if err != nil {
  330. resp.WriteHeader(http.StatusInternalServerError)
  331. resp.Write([]byte(fmt.Sprintf("Cannot create request: %v", err)))
  332. return
  333. }
  334. preq.Header = req.Header
  335. presp, err := bulkClient.Do(preq)
  336. if err != nil {
  337. resp.WriteHeader(http.StatusBadGateway)
  338. resp.Write([]byte(fmt.Sprintf("Cannot obtain data pack from repo host: %v", err)))
  339. return
  340. }
  341. defer presp.Body.Close()
  342. header := resp.Header()
  343. for key, values := range presp.Header {
  344. header[key] = values
  345. }
  346. resp.WriteHeader(presp.StatusCode)
  347. // Ignore errors. Dropped connections are usual and will make this fail.
  348. _, err = io.Copy(resp, presp.Body)
  349. if err != nil {
  350. log.Printf("Error copying data from repo host: %v", err)
  351. }
  352. }
  353. var ErrNoRepo = errors.New("repository not found on repo host")
  354. var ErrNoVersion = errors.New("version reference not found on repo host")
  355. func fetchRefs(repo *Repo) (data []byte, err error) {
  356. resp, err := httpClient.Get("https://" + repo.GitHubRoot() + refsSuffix)
  357. if err != nil {
  358. return nil, fmt.Errorf("cannot talk to repo host: %v", err)
  359. }
  360. defer resp.Body.Close()
  361. switch resp.StatusCode {
  362. case 200:
  363. // ok
  364. case 401, 404:
  365. return nil, ErrNoRepo
  366. default:
  367. return nil, fmt.Errorf("error from repo host: %v", resp.Status)
  368. }
  369. data, err = ioutil.ReadAll(resp.Body)
  370. if err != nil {
  371. return nil, fmt.Errorf("error reading from repo host: %v", err)
  372. }
  373. return data, err
  374. }
  375. func changeRefs(data []byte, major Version) (changed []byte, versions VersionList, err error) {
  376. var hlinei, hlinej int // HEAD reference line start/end
  377. var mlinei, mlinej int // master reference line start/end
  378. var vrefhash string
  379. var vrefname string
  380. var vrefv = InvalidVersion
  381. // Record all available versions, the locations of the master and HEAD lines,
  382. // and details of the best reference satisfying the requested major version.
  383. versions = make([]Version, 0)
  384. sdata := string(data)
  385. for i, j := 0, 0; i < len(data); i = j {
  386. size, err := strconv.ParseInt(sdata[i:i+4], 16, 32)
  387. if err != nil {
  388. return nil, nil, fmt.Errorf("cannot parse refs line size: %s", string(data[i:i+4]))
  389. }
  390. if size == 0 {
  391. size = 4
  392. }
  393. j = i + int(size)
  394. if j > len(sdata) {
  395. return nil, nil, fmt.Errorf("incomplete refs data received from repo host")
  396. }
  397. if sdata[0] == '#' {
  398. continue
  399. }
  400. hashi := i + 4
  401. hashj := strings.IndexByte(sdata[hashi:j], ' ')
  402. if hashj < 0 || hashj != 40 {
  403. continue
  404. }
  405. hashj += hashi
  406. namei := hashj + 1
  407. namej := strings.IndexAny(sdata[namei:j], "\n\x00")
  408. if namej < 0 {
  409. namej = j
  410. } else {
  411. namej += namei
  412. }
  413. name := sdata[namei:namej]
  414. if name == "HEAD" {
  415. hlinei = i
  416. hlinej = j
  417. }
  418. if name == "refs/heads/master" {
  419. mlinei = i
  420. mlinej = j
  421. }
  422. if strings.HasPrefix(name, "refs/heads/v") || strings.HasPrefix(name, "refs/tags/v") {
  423. if strings.HasSuffix(name, "^{}") {
  424. // Annotated tag is peeled off and overrides the same version just parsed.
  425. name = name[:len(name)-3]
  426. }
  427. v, ok := parseVersion(name[strings.IndexByte(name, 'v'):])
  428. if ok && major.Contains(v) && (v == vrefv || !vrefv.IsValid() || vrefv.Less(v)) {
  429. vrefv = v
  430. vrefhash = sdata[hashi:hashj]
  431. vrefname = name
  432. }
  433. if ok {
  434. versions = append(versions, v)
  435. }
  436. }
  437. }
  438. // If there were absolutely no versions, and v0 was requested, accept the master as-is.
  439. if len(versions) == 0 && major == (Version{0, -1, -1, false}) {
  440. return data, nil, nil
  441. }
  442. // If the file has no HEAD line or the version was not found, report as unavailable.
  443. if hlinei == 0 || vrefhash == "" {
  444. return nil, nil, ErrNoVersion
  445. }
  446. var buf bytes.Buffer
  447. buf.Grow(len(data) + 256)
  448. // Copy the header as-is.
  449. buf.Write(data[:hlinei])
  450. // Extract the original capabilities.
  451. caps := ""
  452. if i := strings.Index(sdata[hlinei:hlinej], "\x00"); i > 0 {
  453. caps = strings.Replace(sdata[hlinei+i+1:hlinej-1], "symref=", "oldref=", -1)
  454. }
  455. // Insert the HEAD reference line with the right hash and a proper symref capability.
  456. var line string
  457. if strings.HasPrefix(vrefname, "refs/heads/") {
  458. if caps == "" {
  459. line = fmt.Sprintf("%s HEAD\x00symref=HEAD:%s\n", vrefhash, vrefname)
  460. } else {
  461. line = fmt.Sprintf("%s HEAD\x00symref=HEAD:%s %s\n", vrefhash, vrefname, caps)
  462. }
  463. } else {
  464. if caps == "" {
  465. line = fmt.Sprintf("%s HEAD\n", vrefhash)
  466. } else {
  467. line = fmt.Sprintf("%s HEAD\x00%s\n", vrefhash, caps)
  468. }
  469. }
  470. fmt.Fprintf(&buf, "%04x%s", 4+len(line), line)
  471. // Insert the master reference line.
  472. line = fmt.Sprintf("%s refs/heads/master\n", vrefhash)
  473. fmt.Fprintf(&buf, "%04x%s", 4+len(line), line)
  474. // Append the rest, dropping the original master line if necessary.
  475. if mlinei > 0 {
  476. buf.Write(data[hlinej:mlinei])
  477. buf.Write(data[mlinej:])
  478. } else {
  479. buf.Write(data[hlinej:])
  480. }
  481. return buf.Bytes(), versions, nil
  482. }