odit/npm-cache-proxy
4
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Releases Activity

Compare commits

base: odit:fa1c796f3c2677e8c702d2fc8d1bc7ca44e4e185
Branches Tags
odit:main
odit:renovate/go.uber.org-zap-1.x
..
compare: odit:f821e794f8481d9d580b2b5fb2ed4042ddf1acd3
Branches Tags
odit:renovate/go.uber.org-zap-1.x
odit:main

No commits in common. "fa1c796f3c2677e8c702d2fc8d1bc7ca44e4e185" and "f821e794f8481d9d580b2b5fb2ed4042ddf1acd3" have entirely different histories.
fa1c796f3c ... f821e794f8

12 changed files with 192 additions and 208 deletions
Show Stats Expand all files Collapse all files

10
cli/list.go
View File

@ -3,7 +3,7 @@ package cli
import (
"fmt"
npmproxy "github.com/pkgems/npm-cache-proxy/proxy"
npmproxy "github.com/emeralt/npm-cache-proxy/proxy"
"github.com/spf13/cobra"
)
@ -12,11 +12,13 @@ var listCmd = &cobra.Command{
Use: "list",
Short: "List all cached paths",
Run: func(cmd *cobra.Command, args []string) {
proxy := getProxy()
metadatas, err := proxy.ListCachedPaths(npmproxy.Options{
proxy := getProxy(func() (npmproxy.Options, error) {
return npmproxy.Options{
DatabasePrefix: persistentOptions.RedisPrefix,
}, nil
})
metadatas, err := proxy.ListCachedPaths()
if err != nil {
panic(err)
}

5
cli/main.go
View File

@ -5,7 +5,7 @@ import (
"net/http"
"os"
npmproxy "github.com/pkgems/npm-cache-proxy/proxy"
npmproxy "github.com/emeralt/npm-cache-proxy/proxy"
"github.com/go-redis/redis"
)
@ -25,7 +25,7 @@ func init() {
rootCmd.PersistentFlags().StringVar(&persistentOptions.RedisPrefix, "redis-prefix", getEnvString("REDIS_PREFIX", "ncp-"), "Redis prefix")
}
func getProxy() *npmproxy.Proxy {
func getProxy(getOptions func() (npmproxy.Options, error)) *npmproxy.Proxy {
return &npmproxy.Proxy{
Database: npmproxy.DatabaseRedis{
Client: redis.NewClient(&redis.Options{
@ -37,6 +37,7 @@ func getProxy() *npmproxy.Proxy {
HttpClient: &http.Client{
Transport: http.DefaultTransport,
},
GetOptions: getOptions,
}
}

10
cli/purge.go
View File

@ -1,7 +1,7 @@
package cli
import (
npmproxy "github.com/pkgems/npm-cache-proxy/proxy"
npmproxy "github.com/emeralt/npm-cache-proxy/proxy"
"github.com/spf13/cobra"
)
@ -10,11 +10,13 @@ var purgeCmd = &cobra.Command{
Use: "purge",
Short: "Purge all cached paths",
Run: func(cmd *cobra.Command, args []string) {
proxy := getProxy()
err := proxy.PurgeCachedPaths(npmproxy.Options{
proxy := getProxy(func() (npmproxy.Options, error) {
return npmproxy.Options{
DatabasePrefix: persistentOptions.RedisPrefix,
}, nil
})
err := proxy.PurgeCachedPaths()
if err != nil {
panic(err)
}

23
cli/root.go
View File

@ -1,10 +1,9 @@
package cli
import (
"log"
"time"
npmproxy "github.com/pkgems/npm-cache-proxy/proxy"
npmproxy "github.com/emeralt/npm-cache-proxy/proxy"
"github.com/spf13/cobra"
)
@ -32,24 +31,16 @@ func init() {
}
func run(cmd *cobra.Command, args []string) {
proxy := getProxy()
log.Print("Listening on " + rootOptions.ListenAddress)
err := proxy.Server(npmproxy.ServerOptions{
ListenAddress: rootOptions.ListenAddress,
Silent: rootOptions.Silent,
GetOptions: func() (npmproxy.Options, error) {
proxy := getProxy(func() (npmproxy.Options, error) {
return npmproxy.Options{
DatabasePrefix: persistentOptions.RedisPrefix,
DatabaseExpiration: time.Duration(rootOptions.CacheTTL) * time.Second,
UpstreamAddress: rootOptions.UpstreamAddress,
}, nil
},
}).ListenAndServe()
})
if err != nil {
log.Fatal(err)
}
proxy.Server(npmproxy.ServerOptions{
ListenAddress: rootOptions.ListenAddress,
Silent: rootOptions.Silent,
}).ListenAndServe()
}

10
example/main.go
View File

@ -4,7 +4,7 @@ import (
"net/http"
"time"
npmproxy "github.com/pkgems/npm-cache-proxy/proxy"
npmproxy "github.com/emeralt/npm-cache-proxy/proxy"
"github.com/go-redis/redis"
)
@ -18,10 +18,6 @@ func main() {
}),
},
HttpClient: &http.Client{},
}
proxy.Server(npmproxy.ServerOptions{
ListenAddress: "localhost:8080",
GetOptions: func() (npmproxy.Options, error) {
return npmproxy.Options{
DatabasePrefix: "ncp-",
@ -29,5 +25,9 @@ func main() {
UpstreamAddress: "https://registry.npmjs.org",
}, nil
},
}
proxy.Server(npmproxy.ServerOptions{
ListenAddress: "localhost:8080",
}).ListenAndServe()
}

3
go.mod
View File

@ -1,4 +1,4 @@
module github.com/pkgems/npm-cache-proxy
module github.com/emeralt/npm-cache-proxy
go 1.12
@ -6,7 +6,6 @@ require (
github.com/gin-contrib/zap v0.0.0-20190405225521-7c4b822813e7
github.com/gin-gonic/gin v1.3.0
github.com/go-redis/redis v6.15.2+incompatible
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/spf13/cobra v0.0.3
github.com/spf13/pflag v1.0.3 // indirect
go.uber.org/zap v1.9.1

2
go.sum
View File

@ -9,8 +9,6 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA
github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=

2
main.go
View File

@ -1,7 +1,7 @@
package main
import (
"github.com/pkgems/npm-cache-proxy/cli"
"github.com/emeralt/npm-cache-proxy/cli"
)
func main() {

27
proxy/cache.go
View File

@ -10,7 +10,12 @@ import (
)
// GetCachedPath returns cached upstream response for a given url path.
func (proxy Proxy) GetCachedPath(options Options, path string, request *http.Request) ([]byte, error) {
func (proxy Proxy) GetCachedPath(path string, request *http.Request) ([]byte, error) {
options, err := proxy.GetOptions()
if err != nil {
return nil, err
}
key := options.DatabasePrefix + path
// get package from database
@ -36,6 +41,7 @@ func (proxy Proxy) GetCachedPath(options Options, path string, request *http.Req
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.Header.Get("Content-Encoding") == "gzip" {
zr, err := gzip.NewReader(res.Body)
@ -46,17 +52,14 @@ func (proxy Proxy) GetCachedPath(options Options, path string, request *http.Req
res.Body = zr
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
pkg = string(body)
// TODO: avoid calling MustCompile every time
// find "dist": "https?://.*/ and replace to "dist": "{localurl}/
pkg = regexp.MustCompile(`(?U)"tarball":"https?://.*/`).ReplaceAllString(string(body), `"tarball": "http://`+request.Host+"/")
pkg = regexp.MustCompile(`(?U)"tarball":"https?://.*/`).ReplaceAllString(string(body), `"dist": "http://localhost:8080/`)
// save to redis
err = proxy.Database.Set(key, pkg, options.DatabaseExpiration)
@ -69,7 +72,12 @@ func (proxy Proxy) GetCachedPath(options Options, path string, request *http.Req
}
// ListCachedPaths returns list of all cached url paths.
func (proxy Proxy) ListCachedPaths(options Options) ([]string, error) {
func (proxy Proxy) ListCachedPaths() ([]string, error) {
options, err := proxy.GetOptions()
if err != nil {
return nil, err
}
metadata, err := proxy.Database.Keys(options.DatabasePrefix)
if err != nil {
return nil, err
@ -84,7 +92,12 @@ func (proxy Proxy) ListCachedPaths(options Options) ([]string, error) {
}
// PurgeCachedPaths deletes all cached url paths.
func (proxy Proxy) PurgeCachedPaths(options Options) error {
func (proxy Proxy) PurgeCachedPaths() error {
options, err := proxy.GetOptions()
if err != nil {
return err
}
metadata, err := proxy.Database.Keys(options.DatabasePrefix)
if err != nil {
return err

4
proxy/proxy.go
View File

@ -1,6 +1,6 @@
//
// Package proxy implements a HTTP caching proxy for Node package registry (NPM).
// See https://github.com/pkgems/npm-cache-proxy/ for more information about proxy.
// See https://github.com/emeralt/npm-cache-proxy/ for more information about proxy.
//
package proxy
@ -14,6 +14,8 @@ import (
type Proxy struct {
Database Database
HttpClient *http.Client
GetOptions func() (Options, error)
}
// Options provides dynamic options for Proxy.

46
proxy/server.go
View File

@ -2,7 +2,6 @@ package proxy
import (
"net/http"
"strconv"
"strings"
"time"
@ -15,14 +14,10 @@ import (
type ServerOptions struct {
ListenAddress string
Silent bool
GetOptions func() (Options, error)
}
// Server creates http proxy server
func (proxy Proxy) Server(options ServerOptions) *http.Server {
gin.SetMode("release")
router := gin.New()
if options.Silent {
@ -33,9 +28,9 @@ func (proxy Proxy) Server(options ServerOptions) *http.Server {
router.Use(ginzap.RecoveryWithZap(logger, true))
}
router.GET("/:scope/:name", proxy.getPackageHandler(options))
router.GET("/:scope", proxy.getPackageHandler(options))
router.NoRoute(proxy.noRouteHandler(options))
router.GET("/:scope/:name", proxy.getPackageHandler)
router.GET("/:scope", proxy.getPackageHandler)
router.NoRoute(proxy.noRouteHandler)
return &http.Server{
Handler: router,
@ -43,34 +38,31 @@ func (proxy Proxy) Server(options ServerOptions) *http.Server {
}
}
func (proxy Proxy) getPackageHandler(options ServerOptions) gin.HandlerFunc {
return func(c *gin.Context) {
options, err := options.GetOptions()
func (proxy Proxy) getPackageHandler(c *gin.Context) {
pkg, err := proxy.GetCachedPath(c.Request.URL.Path, c.Request)
if err != nil {
c.AbortWithError(500, err)
} else {
pkg, err := proxy.GetCachedPath(options, c.Request.URL.Path, c.Request)
if err != nil {
c.AbortWithError(500, err)
} else {
c.Header("Cache-Control", "public, max-age="+strconv.Itoa(int(options.DatabaseExpiration.Seconds())))
// c.Header("Content-Encoding", "gzip")
c.Data(200, "application/json", pkg)
}
}
}
}
func (proxy Proxy) noRouteHandler(options ServerOptions) gin.HandlerFunc {
tarballHandler := proxy.getPackageHandler(options)
func (proxy Proxy) getTarballHabdler(c *gin.Context) {
pkg, err := proxy.GetCachedPath(c.Request.URL.Path, c.Request)
return func(c *gin.Context) {
if err != nil {
c.AbortWithError(500, err)
} else {
c.Data(200, "application/json", pkg)
}
}
func (proxy Proxy) noRouteHandler(c *gin.Context) {
if strings.Contains(c.Request.URL.Path, ".tgz") {
// get tarball
tarballHandler(c)
proxy.getTarballHabdler(c)
} else if c.Request.URL.Path == "/" {
// get health
err := proxy.Database.Health()
if err != nil {
@ -79,8 +71,7 @@ func (proxy Proxy) noRouteHandler(options ServerOptions) gin.HandlerFunc {
c.AbortWithStatusJSON(200, gin.H{"ok": true})
}
} else {
// redirect
options, err := options.GetOptions()
options, err := proxy.GetOptions()
if err != nil {
c.AbortWithStatusJSON(500, err)
@ -88,5 +79,4 @@ func (proxy Proxy) noRouteHandler(options ServerOptions) gin.HandlerFunc {
c.Redirect(http.StatusTemporaryRedirect, options.UpstreamAddress+c.Request.URL.Path)
}
}
}
}

202
readme.md
View File

@ -1,135 +1,104 @@
<div align="center">
<img width="450" src="./logo.png">
<h1 align="center">
<img width="400" src="./logo.png">
<h1>npm-cache-proxy</h1>
npm-cache-proxy
</h1>
<a href="https://hub.docker.com/r/pkgems/npm-cache-proxy/tags">
<img src="https://img.shields.io/github/release/pkgems/npm-cache-proxy.svg" alt="Current Release" />
<p align="center">
<a href="https://hub.docker.com/r/emeralt/npm-cache-proxy/tags">
<img src="https://img.shields.io/github/release/emeralt/npm-cache-proxy.svg" alt="Current Release" />
</a>
<a href="https://hub.docker.com/r/pkgems/npm-cache-proxy/builds">
<img src="https://img.shields.io/docker/cloud/build/pkgems/npm-cache-proxy.svg" alt="CI Build">
<a href="https://hub.docker.com/r/emeralt/npm-cache-proxy/builds">
<img src="https://img.shields.io/docker/cloud/build/emeralt/npm-cache-proxy.svg" alt="CI Build">
</a>
<a href="https://github.com/pkgems/npm-cache-proxy/blob/master/license">
<img src="https://img.shields.io/github/license/pkgems/npm-cache-proxy.svg" alt="Licence">
<a href="https://github.com/emeralt/npm-cache-proxy/blob/master/liscense">
<img src="https://img.shields.io/github/license/emeralt/npm-cache-proxy.svg" alt="Licence">
</a>
</div>
</p>
<br />
<br />
## Introduction
#### ⚡️ Performance
NCP is a tiny but very fast caching proxy written in Go. It uses Redis for data storage, which in combination with the speed of Go makes it incredibly fast. NCP is well-optimized and can be run on almost any platform, so if you have a Raspberry Pi, you can install NCP as your local cache there.
#### ✨ Modularity
NCP is modular. Now it has only one database adapter which is Redis. If you need support for any other database, feel free to open an issue or implement it [on your own](https://github.com/pkgems/npm-cache-proxy/blob/7c8b90ff6ba0656f60e3de915b9fb4eaabfb467b/proxy/proxy.go#L29) and then open a pull request (_bonus points_).
#### 💡 Simplicity
NCP is very simple. It just proxies requests to an upstream registry, caches response and returns cached response for next requests to the same package. Cached data are stored in Redis with an original request URL as a key.
> `npm-cache-proxy` is a lightweight npm caching proxy written in Go that achieves warp speeds by using Redis for data storage. B-b-blazing fast!
<br />
<details>
<summary>Table of Contents</summary>
<p>
- [Getting started](#getting-started)
- [CLI](#cli)
- [`ncp`](#ncp)
- [`ncp list`](#ncp-list)
- [`ncp purge`](#ncp-purge)
- [Programmatic usage](#programmatic-usage)
- [Example](#example)
- [Deployment](#deployment)
- [Benchmark](#benchmark)
- [License](#license)
## Installation
NCP binaries for different paltforms can be downloaded can be downloaded on the [Releases](https://github.com/pkgems/npm-cache-proxy/releases) page. Also, Docker image is provided on [Docker Hub](https://cloud.docker.com/u/pkgems/repository/docker/pkgems/npm-cache-proxy).
</p>
</details>
#### 💫 Quick Start
The quickies way to get started with NCP is to use Docker.
## Getting started
Release binaries for different platforms can be downloaded on the [Releases](https://github.com/emeralt/npm-cache-proxy/releases) page. Also, [Docker image](https://cloud.docker.com/u/emeralt/repository/docker/emeralt/npm-cache-proxy) is provided.
The fastest way to get started with NCP is to use Docker image:
```bash
# run proxy inside of docker container in background
docker run -e REDIS_ADDRESS=host.docker.internal:6379 -p 8080:8080 -it -d pkgems/npm-cache-proxy
docker run -e REDIS_ADDRESS=host.docker.internal:6379 -p 8080:8080 -it emeralt/npm-cache-proxy -d
# configure npm to use caching proxy as registry
npm config set registry http://localhost:8080
```
<br />
## CLI
NCP provides command line interface for interaction with a cached data.
Additionally, NCP provides a command line utility for proxy configuration and data management.
<details>
<summary>Options</summary>
---
| Options | Env | Default | Description |
| ----------------------------- | ------------------ | ---------------------------- | ----------------------------------- |
| `--listen <address>` | `LISTEN_ADDRESS` | `locahost:8080` | Address to listen |
| `--upstream <address>` | `UPSTREAM_ADDRESS` | `https://registry.npmjs.org` | Upstream registry address |
| `--silent <address>` | `SILENT` | `0` | Disable logs |
| `--cache-limit <count>` | `CACHE_LIMIT` | - | Cached packages count limit |
| `--cache-ttl <timeout>` | `CACHE_TTL` | `3600` | Cache expiration timeout in seconds |
| Global Options | Env | Default | Description |
| ----------------------------- | ---------------- | ----------------------- | ----------------- |
| `--redis-address <address>` | `REDIS_ADDRESS` | `http://localhost:6379` | Redis address |
| `--redis-database <database>` | `REDIS_DATABASE` | `0` | Redis database |
| `--redis-password <password>` | `REDIS_PASSWORD` | - | Redis password |
| `--redis-prefix <prefix>` | `REDIS_PREFIX` | `ncp-` | Redis keys prefix |
</details>
---
#### `ncp`
Start NCP server.
### `ncp`
#### `ncp list`
List cached url paths.
#### `ncp purge`
Purge cached url paths.
<br />
## Benchmark
Benchmark is run on Macbook Pro 15″ 2017, Intel Core i7-7700HQ.
#### 1️⃣ 1 process
Start proxy server.
```bash
# GOMAXPROCS=1 ncp --silent
$ go-wrk -c 100 -d 6 http://localhost:8080/tiny-tarball
Running 6s test @ http://localhost:8080/tiny-tarball
100 goroutine(s) running concurrently
70755 requests in 5.998378587s, 91.16MB read
Requests/sec: 11795.69
Transfer/sec: 15.20MB
Avg Req Time: 8.477674ms
Fastest Request: 947.743µs
Slowest Request: 815.787409ms
Number of Errors: 0
ncp --listen "localhost:1234" # listen on port 1234
```
#### ♾ unlimited processes
| Options | Env | Default | Description |
| ----------------------- | ------------------ | ---------------------------- | ----------------------------------- |
| `--listen <address>` | `LISTEN_ADDRESS` | `locahost:8080` | Address to listen |
| `--upstream <address>` | `UPSTREAM_ADDRESS` | `https://registry.npmjs.org` | Upstream registry address |
| `--cache-limit <count>` | `CACHE_LIMIT` | - | Cached packages count limit |
| `--cache-ttl <timeout>` | `CACHE_TTL` | `3600` | Cache expiration timeout in seconds |
| `--silent <address>` | `SILENT` | `0` | Disable logs |
---
### `ncp list`
List all cached packages.
---
### `ncp purge`
Purge all cached packages.
---
## Programmatic usage
Along with the CLI, go package is provided. Documentation is available on [godoc.org](https://godoc.org/github.com/emeralt/npm-cache-proxy/proxy).
```bash
# ncp --silent
$ go-wrk -c 100 -d 6 http://localhost:8080/tiny-tarball
Running 6s test @ http://localhost:8080/tiny-tarball
100 goroutine(s) running concurrently
115674 requests in 5.98485984s, 149.04MB read
Requests/sec: 19327.77
Transfer/sec: 24.90MB
Avg Req Time: 5.173902ms
Fastest Request: 273.015µs
Slowest Request: 34.777963ms
Number of Errors: 0
go get -u github.com/emeralt/npm-cache-proxy/proxy
```
<br />
## Programmatic Usage
NCP provides `proxy` go package that can be used programmatically. Docs are available on [godoc.org](https://godoc.org/github.com/pkgems/npm-cache-proxy/proxy).
#### 🤖 Example
### Example
```golang
package main
@ -137,30 +106,24 @@ import (
"net/http"
"time"
npmproxy "github.com/pkgems/npm-cache-proxy/proxy"
"github.com/go-redis/redis"
npmproxy "github.com/emeralt/npm-cache-proxy/proxy"
redis "github.com/go-redis/redis"
)
func main() {
// create proxy
proxy := npmproxy.Proxy{
// use redis as database
// you can provide you own Database
// or use an existing one
Database: npmproxy.DatabaseRedis{
// see github.com/go-redis/redis
Client: redis.NewClient(&redis.Options{
Addr: "localhost:6379",
}),
},
// reuse connections
// allows to reuse tcp connections
HttpClient: &http.Client{},
}
// create and start server
proxy.Server(npmproxy.ServerOptions{
ListenAddress: "localhost:8080",
// allow fetching options dynamically on each request
// allows to get options dynamically
GetOptions: func() (npmproxy.Options, error) {
return npmproxy.Options{
DatabasePrefix: "ncp-",
@ -168,13 +131,36 @@ func main() {
UpstreamAddress: "https://registry.npmjs.org",
}, nil
},
}
// listen on http://localhost:8080
proxy.Server(npmproxy.ServerOptions{
ListenAddress: "localhost:8080",
}).ListenAndServe()
}
```
## Deployment
NCP can be deployed using Kubernetes, Docker Compose or any other container orchestration platform. NCP supports running indefinite amount of instances simultaneously.
<br />
## Benchmark
Macbook Pro 15″ 2017, Intel Core i7-7700HQ. Note `GOMAXPROCS=1`.
```bash
# SILENT=1 GIN_MODE=release GOMAXPROCS=1 ncp
$ go-wrk -c 100 -d 10 http://localhost:8080/ascii
Running 10s test @ http://localhost:8080/ascii
100 goroutine(s) running concurrently
84216 requests in 10.000196326s, 535.30MB read
Requests/sec: 8421.43
Transfer/sec: 53.53MB
Avg Req Time: 11.874461ms
Fastest Request: 2.213324ms
Slowest Request: 745.874068ms
Number of Errors: 0
```
## License
[MIT](./license)