This post describes how you can write your own file-based cache in Golang, similar to Nginx, that can cache anything from frequently used data from the database to data obtained through requests and also survive application restarts.
Project Setup:
We will start by initialising a go.mod file in the working directory of the project
go mod init <MODULE_NAME>The next step is to create a main.go file with the following contents
package main
func main() {
}The Main Idea:
The application we will be building will handle HTTP requests to download some large images from the internet.
The application makes the request on behalf of the client and stores the image on the disc with a UUID name. Then the download URL is mapped to this UUID in a persistent key-value store.
Then, when a new request comes, we first check if the URL is already present in the key-value store, and if it is, we just respond with the correct image from the disk. Otherwise, we make a request to the URL and cache the image on the disk.
We will be using a couple of third party libraries to make our lives simpler,
github.com/julienschmidt/httprouter — to handle http request
git.mills.io/prologic/bitcask — a persistent key-value store
github.com/google/uuid — library to generate UUID
Handling HTTP Requests:
We will start by writing a basic HTTP server that gets the image URL from the request, makes a GET request to get the image, and then sends it back to the client.
package main
import (
"io/ioutil"
"log"
"net/http"
"github.com/julienschmidt/httprouter"
)
func handleDownload(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
queryValues := r.URL.Query()
url := queryValues.Get("url")
resp, err := http.Get(url)
if err != nil {
log.Fatalln(err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
log.Println(body)
w.Header().Set("Content-Type", "image/png")
w.Write(body)
}
func main() {
router := httprouter.New()
router.GET("/cache", handleCache)
log.Fatalln(http.ListenAndServe(":8080", router))
}Now, if you run this program and visit localhost:8080/cache?url=”some_image_url”, you should see the image appear.
Example : http://localhost:8080/cache?url=https://images.unsplash.com/photo-1526666923127-b2970f64b422?ixlib=rb-4.0.3&ixid=MnwxMjA3fDB8MHxzZWFyY2h8MXx8ZXh0cmElMjBsYXJnZXxlbnwwfHwwfHw%3D&w=1000&q=80
Note: Make sure to run go mod tidy
Caching the Results:
Let’s define a new function, checkCacheFolder().
func checkCacheFolder() {
if _, err := os.Stat(".cache"); errors.Is(err, os.ErrNotExist) {
err := os.Mkdir(".cache", os.ModePerm)
if err != nil {
log.Println(err)
}
}
}
func main() {
checkCacheFolder()
...
}This new function checks if the hidden directory cache already exists. If not, it creates it. All our cache images will be stored here. Feel free to change where the cache folder gets created. /tmp or /usr is usually preferred.
Now, we will define the function which handles the caching part.
func getKeyHash(url string) string {
keyHash := fnv.New64()
keyHash.Write([]byte(url))
return fmt.Sprintf("%v", keyHash.Sum64())
}
func cacheImages(img []byte, url string) {
key := getKeyHash(url)
if db.Has([]byte(key)) {
return
}
id := uuid.New().String()
if _, err := os.Stat(".cache/" + id); errors.Is(err, os.ErrNotExist) {
_, err := os.Create(".cache/" + id)
if err != nil {
log.Println(err)
}
err = os.WriteFile(id, img, 0644)
if err != nil {
log.Println(err)
}
}
db.Put([]byte(key), []byte(id))
}The function cacheImages basically gets the image as a byte array and writes it to a file inside the cache directory.
The next step is to add this to the KeyValue store. The function getKeyHash converts the URL into a 64 bit hash key, which can be efficiently stored in the Bitcask key-value store.
We then map this key to the UUID of the image file.
Refactoring and Final Code:
Now, all that’s left is to check if the key already exists in the handleDownload function and return the correct image from the cache if it does. With all the changes, the final code looks like this:
package main
import (
"errors"
"fmt"
"hash/fnv"
"io/ioutil"
"log"
"net/http"
"os"
"git.mills.io/prologic/bitcask"
"github.com/google/uuid"
"github.com/julienschmidt/httprouter"
)
var db *bitcask.Bitcask
func checkCacheFolder() {
if _, err := os.Stat(".cache"); errors.Is(err, os.ErrNotExist) {
err := os.Mkdir(".cache", os.ModePerm)
if err != nil {
log.Println(err)
}
}
}
func getKeyHash(url string) string {
keyHash := fnv.New64()
keyHash.Write([]byte(url))
return fmt.Sprintf("%v", keyHash.Sum64())
}
func cacheImages(img []byte, url string) {
key := getKeyHash(url)
if db.Has([]byte(key)) {
return
}
id := uuid.New().String()
if _, err := os.Stat(".cache/" + id); errors.Is(err, os.ErrNotExist) {
_, err := os.Create(".cache/" + id)
if err != nil {
log.Println(err)
}
err = os.WriteFile(id, img, 0644)
if err != nil {
log.Println(err)
}
}
db.Put([]byte(key), []byte(id))
}
func handleCache(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
queryValues := r.URL.Query()
url := queryValues.Get("url")
key := getKeyHash(url))
if db.Has([]byte(key)) {
log.Println("cache hit!")
val, err := db.Get([]byte(key))
if err != nil {
log.Fatalln(err)
}
img, err := os.ReadFile(".cache/" + string(val))
if err != nil {
log.Fatalln(err)
}
w.Header().Set("Content-Type", "image/png")
w.Write(img)
}
resp, err := http.Get(url)
if err != nil {
log.Fatalln(err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
cacheImages(body, url)
w.Header().Set("Content-Type", "image/png")
w.Write(body)
}
func main() {
checkCacheFolder()
db, err := bitcask.Open("/tmp/db")
if err != nil {
log.Fatalln(err)
}
defer db.Close()
router := httprouter.New()
router.GET("/cache", handleCache)
log.Fatalln(http.ListenAndServe(":8080", router))
}
