Cold Storage with One-Way RSA Encryption #43

Merged
noah merged 2 commits from coldstorage into main 2024-05-31 00:01:32 +00:00
13 changed files with 569 additions and 65 deletions

1
.gitignore vendored
View File

@ -1,5 +1,6 @@
/nonshy
/web/static/photos
/coldstorage
database.sqlite
settings.json
pgdump.sql

View File

@ -6,6 +6,7 @@ import (
nonshy "code.nonshy.com/nonshy/website/pkg"
"code.nonshy.com/nonshy/website/pkg/config"
"code.nonshy.com/nonshy/website/pkg/encryption/coldstorage"
"code.nonshy.com/nonshy/website/pkg/log"
"code.nonshy.com/nonshy/website/pkg/models"
"code.nonshy.com/nonshy/website/pkg/models/backfill"
@ -169,6 +170,57 @@ func main() {
},
},
},
{
Name: "coldstorage",
Usage: "cold storage functions for sensitive files",
Subcommands: []*cli.Command{
{
Name: "decrypt",
Usage: "decrypt a file from cold storage using the RSA private key",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "key",
Aliases: []string{"k"},
Required: true,
Usage: "RSA private key file for cold storage",
},
&cli.StringFlag{
Name: "aes",
Aliases: []string{"a"},
Required: true,
Usage: "AES key file used with the encrypted item in question (.aes file)",
},
&cli.StringFlag{
Name: "input",
Aliases: []string{"i"},
Required: true,
Usage: "input file to decrypt (.enc file)",
},
&cli.StringFlag{
Name: "output",
Aliases: []string{"o"},
Required: true,
Usage: "output file to write to (like a .jpg file)",
},
},
Action: func(c *cli.Context) error {
err := coldstorage.FileFromColdStorage(
c.String("key"),
c.String("aes"),
c.String("input"),
c.String("output"),
)
if err != nil {
log.Error("Error decrypting from cold storage: %s", err)
return err
}
log.Info("Wrote decrypted file to: %s", c.String("output"))
return nil
},
},
},
},
{
Name: "backfill",
Usage: "One-off maintenance tasks and data backfills for database migrations",

70
docs/Cold Storage.md Normal file
View File

@ -0,0 +1,70 @@
# Cold Storage
One of the security features of the website is **cold storage** which implements a "one way" encryption process for archiving sensitive files on the site.
The first use case is to archive secondary photo IDs: if a user was requested to provide a scan of their government issued photo ID for approval, the site can archive the original copy to cold storage when approved in case of any future inquiry.
The cold storage feature works by encrypting the file using an RSA public key, and relies on the matching private key to be **removed** from the web server and kept offline; so in case of a hack or data breach, the key that can decrypt the cold storage files will **NOT** be kept on the same web server.
This document explains how this feature works and how to configure it.
## Initialization
When the server starts up and there is not a cold storage RSA key configured, the feature will be initialized by generating new RSA encryption keys:
* The directory `./coldstorage/keys` is created and the RSA keys will be written in files named **private.pem** and **public.pem**.
* The RSA public key is _also_ written into the **settings.json** file for the server, at the Encryption / ColdStorageRSAPublicKey property.
You should **move the keys OFF of your web server machine** and keep them safe for your bookkeeping. Notably, the `private.pem` key is the sensitive file that should be removed.
The app does not need either of these keys to remain on the server: the settings.json has a copy of the RSA public key which the app uses to create cold storage encrypted files.
### Admin Dashboard Warning
As a safety precaution: if the private.pem key remains on disk, a warning is shown at the top of the Admin Dashboard page of the website to remind you that the key should be removed and stored safely offline.
## Recovering from Cold Storage
Should you need to recover an encrypted file from cold storage, the `nonshy coldstorage decrypt` command built into the Go server binary has the function to decrypt the files.
Every item that is moved into cold storage generates two files: an encrypted AES key file (`.aes`) and the encrypted data file itself (with a `.enc` extension). For example, a "photo.jpg" might go into cold storage as two files: "photo.jpg.aes" and "photo.jpg.enc"
You will need the following three files to decrypt from cold storage:
1. The RSA private key file (private.pem)
2. The encrypted AES key file (.aes extension)
3. The encrypted cold storage data file (.enc extension)
The command to decrypt them is thus like:
```bash
# command example
nonshy coldstorage decrypt \
--key private.pem \
--aes photo.jpg.aes \
--input photo.jpg.enc \
--output photo.jpg
# short command line flags work too
nonshy coldstorage decrypt -k private.pem \
-a photo.jpg.aes -i photo.jpg.enc \
-o photo.jpg
```
The `--output` file is where the decrypted file will be written to.
## Encryption Algorithms
When a file is moved into cold storage:
1. A fresh new AES symmetric key is generated from scratch.
2. The AES key is encrypted using the **RSA public key** and written to the ".aes" file in the coldstorage/ folder.
3. The original file is encrypted using that AES symmetric key and written to the ".enc" file in the coldstorage/ folder.
At the end of the encrypt function: the web server no longer has the AES key and is _unable_ to decrypt it because the private key is not available (as it should be kept offline for security).
Decrypting a file out of cold storage is done like so:
1. The encrypted AES key is unlocked using the **RSA private key**.
2. The encrypted cold storage file (.enc) is decrypted with that AES key.
3. The cleartext data is written to the output file.

View File

@ -6,6 +6,7 @@ import (
"fmt"
"os"
"code.nonshy.com/nonshy/website/pkg/encryption/coldstorage"
"code.nonshy.com/nonshy/website/pkg/encryption/keygen"
"code.nonshy.com/nonshy/website/pkg/log"
"github.com/google/uuid"
@ -13,7 +14,7 @@ import (
// Version of the config format - when new fields are added, it will attempt
// to write the settings.toml to disk so new defaults populate.
var currentVersion = 3
var currentVersion = 4
// Current loaded settings.json
var Current = DefaultVariable()
@ -97,6 +98,19 @@ func LoadSettings() {
writeSettings = true
}
// Initialize the cold storage RSA keys.
if len(Current.Encryption.ColdStorageRSAPublicKey) == 0 {
x509publicKey, err := coldstorage.Initialize()
if err != nil {
log.Error("Initializing cold storage: %s", err)
os.Exit(1)
}
// Store the public key in the settings.json.
Current.Encryption.ColdStorageRSAPublicKey = x509publicKey
writeSettings = true
}
// Have we added new config fields? Save the settings.json.
if Current.Version != currentVersion || writeSettings {
log.Warn("New options are available for your settings.json file. Your settings will be re-saved now.")
@ -163,7 +177,8 @@ type Maintenance struct {
// Encryption settings.
type Encryption struct {
AESKey []byte
AESKey []byte
ColdStorageRSAPublicKey []byte
}
// Turnstile (Cloudflare CAPTCHA) settings.

View File

@ -3,6 +3,7 @@ package admin
import (
"net/http"
"code.nonshy.com/nonshy/website/pkg/encryption/coldstorage"
"code.nonshy.com/nonshy/website/pkg/templates"
)
@ -10,7 +11,10 @@ import (
func Dashboard() http.HandlerFunc {
tmpl := templates.Must("admin/dashboard.html")
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := tmpl.Execute(w, r, nil); err != nil {
var vars = map[string]interface{}{
"ColdStorageWarning": coldstorage.Warning(),
}
if err := tmpl.Execute(w, r, vars); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

View File

@ -7,9 +7,11 @@ import (
"net/http"
"path/filepath"
"strconv"
"time"
"code.nonshy.com/nonshy/website/pkg/chat"
"code.nonshy.com/nonshy/website/pkg/config"
"code.nonshy.com/nonshy/website/pkg/encryption/coldstorage"
"code.nonshy.com/nonshy/website/pkg/geoip"
"code.nonshy.com/nonshy/website/pkg/log"
"code.nonshy.com/nonshy/website/pkg/mail"
@ -468,6 +470,25 @@ func AdminCertification() http.HandlerFunc {
// With a secondary photo ID?
if cert.SecondaryFilename != "" {
// Move the original photo into cold storage.
coldStorageFilename := fmt.Sprintf(
"photoID-%d-%s-%d.jpg",
user.ID,
user.Username,
time.Now().Unix(),
)
if err := coldstorage.FileToColdStorage(
photo.DiskPath(cert.SecondaryFilename),
coldStorageFilename,
config.Current.Encryption.ColdStorageRSAPublicKey,
); err != nil {
session.FlashError(w, r, "Failed to move to cold storage: %s", err)
templates.Redirect(w, r.URL.Path)
return
} else {
session.Flash(w, r, "Note: the secondary photo ID was encrypted to cold storage @ %s", coldStorageFilename)
}
// Delete it immediately.
if err := photo.Delete(cert.SecondaryFilename); err != nil {
session.FlashError(w, r, "Failed to delete old secondary ID cert photo for %s (%s): %s", currentUser.Username, cert.SecondaryFilename, err)

View File

@ -0,0 +1,150 @@
package coldstorage
import (
"crypto/rand"
"crypto/rsa"
"errors"
"fmt"
"os"
"path"
"path/filepath"
"code.nonshy.com/nonshy/website/pkg/encryption/keygen"
"code.nonshy.com/nonshy/website/pkg/log"
)
var (
ColdStorageDirectory = "./coldstorage"
ColdStorageKeysDirectory = path.Join(ColdStorageDirectory, "keys")
ColdStoragePrivateKeyFile = path.Join(ColdStorageKeysDirectory, "private.pem")
ColdStoragePublicKeyFile = path.Join(ColdStorageKeysDirectory, "public.pem")
)
// Initialize generates the RSA key pairs for the first time and creates
// the cold storage directories. It writes the keys to disk and returns the x509 encoded
// public key which goes in the settings.json (the keys on disk are for your bookkeeping).
func Initialize() ([]byte, error) {
log.Warn("NOTICE: rolling a random RSA key pair for cold storage")
rsaKey, err := keygen.NewRSAKeys()
if err != nil {
return nil, fmt.Errorf("generate RSA key: %s", err)
}
// Encode to x509
x509, err := keygen.SerializePublicKey(rsaKey.Public())
if err != nil {
return nil, fmt.Errorf("encode RSA public key to x509: %s", err)
}
// Write the public/private key files to disk.
if _, err := os.Stat(ColdStorageKeysDirectory); os.IsNotExist(err) {
log.Info("Notice: creating cold storage directory")
if err := os.MkdirAll(ColdStorageKeysDirectory, 0755); err != nil {
return nil, fmt.Errorf("create %s: %s", ColdStorageKeysDirectory, err)
}
}
if err := keygen.WriteRSAKeys(
rsaKey,
path.Join(ColdStorageKeysDirectory, "private.pem"),
path.Join(ColdStorageKeysDirectory, "public.pem"),
); err != nil {
return nil, fmt.Errorf("export newly generated public/private key files: %s", err)
}
return x509, nil
}
// Warning returns an error message if the private key is still on disk at its
// original generated location: it should be moved offline for security.
func Warning() error {
if _, err := os.Stat(ColdStoragePrivateKeyFile); os.IsNotExist(err) {
return nil
}
return errors.New("the private key file at ./coldstorage/keys should be moved off of the server and kept offline for safety")
}
// FileToColdStorage will copy a file, encrypted, into cold storage at the given filename.
func FileToColdStorage(sourceFilePath, outputFileName string, publicKeyPEM []byte) error {
if len(publicKeyPEM) == 0 {
return errors.New("no RSA public key")
}
// Load the public key from PEM encoding.
publicKey, err := keygen.DeserializePublicKey(publicKeyPEM)
if err != nil {
return fmt.Errorf("deserializing public key: %s", err)
}
// Generate a unique AES key for encrypting this file in one direction.
aesKey, err := keygen.NewAESKey()
if err != nil {
return err
}
// Encrypt the AES key and store it on disk next to the cold storage file.
ciphertext, err := rsa.EncryptPKCS1v15(rand.Reader, publicKey, aesKey)
if err != nil {
return fmt.Errorf("encrypt error: %s", err)
}
err = os.WriteFile(
filepath.Join(ColdStorageDirectory, outputFileName+".aes"),
ciphertext,
0600,
)
if err != nil {
return err
}
// Read the original plaintext file going into cold storage.
plaintext, err := os.ReadFile(sourceFilePath)
if err != nil {
return fmt.Errorf("source file: %s", err)
}
// Encrypt the plaintext with the AES key.
ciphertext, err = keygen.EncryptWithAESKey(plaintext, aesKey)
if err != nil {
return err
}
// Write it to disk.
return os.WriteFile(filepath.Join(ColdStorageDirectory, outputFileName+".enc"), ciphertext, 0600)
}
// FileFromColdStorage decrypts a cold storage file and writes it to the output file.
//
// The command `nonshy coldstorage decrypt` uses this function. Requirements:
//
// - privateKeyFile is the RSA private key originally generated for cold storage
// - aesKeyFile is the unique .aes file for the cold storage item
// - ciphertextFile is the encrypted cold storage item
// - outputFile is where you want to save the result to
func FileFromColdStorage(privateKeyFile, aesKeyFile, ciphertextFile, outputFile string) error {
privateKey, err := keygen.PrivateKeyFromFile(privateKeyFile)
if err != nil {
return fmt.Errorf("private key file: %s", err)
}
encryptedAESKey, err := os.ReadFile(aesKeyFile)
if err != nil {
return fmt.Errorf("reading aes key file: %s", err)
}
aesKey, err := rsa.DecryptPKCS1v15(rand.Reader, privateKey, encryptedAESKey)
if err != nil {
return fmt.Errorf("decrypting the aes key file: %s", err)
}
ciphertext, err := os.ReadFile(ciphertextFile)
if err != nil {
return fmt.Errorf("reading cold storage file: %s", err)
}
plaintext, err := keygen.DecryptWithAESKey(ciphertext, aesKey)
if err != nil {
return fmt.Errorf("decrypting cold storage file: %s", err)
}
return os.WriteFile(outputFile, plaintext, 0644)
}

View File

@ -7,15 +7,12 @@
package encryption
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"errors"
"fmt"
"io"
"code.nonshy.com/nonshy/website/pkg/config"
"code.nonshy.com/nonshy/website/pkg/encryption/keygen"
)
// Encrypt a byte stream using the site's AES passphrase.
@ -24,32 +21,7 @@ func Encrypt(input []byte) ([]byte, error) {
return nil, errors.New("AES key not configured")
}
// Generate a new AES cipher.
c, err := aes.NewCipher(config.Current.Encryption.AESKey)
if err != nil {
return nil, err
}
// gcm or Galois/Counter Mode
gcm, err := cipher.NewGCM(c)
if err != nil {
return nil, err
}
// Create a new byte array the size of the GCM nonce
// which must be passed to Seal.
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, fmt.Errorf("populating the nonce: %s", err)
}
// Encrypt the text using the Seal function.
// Seal encrypts and authenticates plaintext, authenticates the
// additional data and appends the result to dst, returning the
// updated slice. The nonce must be NonceSize() bytes long and
// unique for all time, for a given key.
result := gcm.Seal(nonce, nonce, input, nil)
return result, nil
return keygen.EncryptWithAESKey(input, config.Current.Encryption.AESKey)
}
// EncryptString encrypts a string value and returns the cipher text.
@ -63,27 +35,7 @@ func Decrypt(data []byte) ([]byte, error) {
return nil, errors.New("AES key not configured")
}
c, err := aes.NewCipher(config.Current.Encryption.AESKey)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(c)
if err != nil {
return nil, err
}
nonceSize := gcm.NonceSize()
if len(data) < nonceSize {
return nil, errors.New("ciphertext data less than nonceSize")
}
nonce, ciphertext := data[:nonceSize], data[nonceSize:]
plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return nil, err
}
return plaintext, nil
return keygen.DecryptWithAESKey(data, config.Current.Encryption.AESKey)
}
// DecryptString decrypts a string value from ciphertext.

View File

@ -0,0 +1,72 @@
// Package keygen provides the AES key initializer function.
package keygen
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"errors"
"fmt"
"io"
)
// NewAESKey returns a 32-byte (AES 256 bit) encryption key.
func NewAESKey() ([]byte, error) {
var result = make([]byte, 32)
_, err := rand.Read(result)
return result, err
}
// EncryptWithAESKey a byte stream using a given AES key.
func EncryptWithAESKey(input []byte, key []byte) ([]byte, error) {
// Generate a new AES cipher.
c, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
// gcm or Galois/Counter Mode
gcm, err := cipher.NewGCM(c)
if err != nil {
return nil, err
}
// Create a new byte array the size of the GCM nonce
// which must be passed to Seal.
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, fmt.Errorf("populating the nonce: %s", err)
}
// Encrypt the text using the Seal function.
// Seal encrypts and authenticates plaintext, authenticates the
// additional data and appends the result to dst, returning the
// updated slice. The nonce must be NonceSize() bytes long and
// unique for all time, for a given key.
result := gcm.Seal(nonce, nonce, input, nil)
return result, nil
}
func DecryptWithAESKey(data []byte, key []byte) ([]byte, error) {
c, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(c)
if err != nil {
return nil, err
}
nonceSize := gcm.NonceSize()
if len(data) < nonceSize {
return nil, errors.New("ciphertext data less than nonceSize")
}
nonce, ciphertext := data[:nonceSize], data[nonceSize:]
plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return nil, err
}
return plaintext, nil
}

View File

@ -0,0 +1,72 @@
package keygen_test
import (
"testing"
"code.nonshy.com/nonshy/website/pkg/encryption/keygen"
)
func TestAES(t *testing.T) {
type testCase struct {
AESKey []byte // AES key, nil = generate a new one
Input []byte // input text to encrypt
Encrypted []byte // already encrypted text
Expect []byte // expected output on decrypt
}
var tests = []testCase{
{
Input: []byte("hello world"),
Expect: []byte("hello world"),
},
{
AESKey: []byte{170, 94, 243, 132, 85, 247, 149, 238, 245, 39, 140, 125, 226, 178, 134, 161, 17, 151, 139, 248, 16, 94, 165, 8, 102, 238, 214, 183, 86, 138, 219, 52},
Encrypted: []byte{146, 217, 250, 254, 70, 201, 27, 221, 92, 145, 77, 213, 211, 197, 63, 189, 220, 188, 78, 8, 217, 108, 136, 89, 156, 23, 179, 54, 209, 54, 244, 170, 182, 150, 242, 52, 112, 191, 216, 46},
Expect: []byte("goodbye mars"),
},
}
for i, test := range tests {
if len(test.AESKey) == 0 {
key, err := keygen.NewAESKey()
if err != nil {
t.Errorf("Test #%d: failed to generate new AES key: %s", i, err)
continue
}
test.AESKey = key
}
if len(test.Encrypted) == 0 {
enc, err := keygen.EncryptWithAESKey(test.Input, test.AESKey)
if err != nil {
t.Errorf("Test #%d: failed to encrypt input: %s", i, err)
continue
}
test.Encrypted = enc
}
// t.Errorf("Key: %+v\nEnc: %+v", test.AESKey, test.Encrypted)
dec, err := keygen.DecryptWithAESKey(test.Encrypted, test.AESKey)
if err != nil {
t.Errorf("Test #%d: failed to decrypt: %s", i, err)
continue
}
// compare the results
var ok = true
if len(dec) != len(test.Expect) {
ok = false
} else {
for j := range dec {
if test.Expect[j] != dec[j] {
ok = false
}
}
}
if !ok {
t.Errorf("Test #%d: got unexpected result from decrypt. Expected %s, got %s", i, test.Expect, dec)
continue
}
}
}

View File

@ -1,11 +0,0 @@
// Package keygen provides the AES key initializer function.
package keygen
import "crypto/rand"
// NewAESKey returns a 32-byte (AES 256 bit) encryption key.
func NewAESKey() ([]byte, error) {
var result = make([]byte, 32)
_, err := rand.Read(result)
return result, err
}

View File

@ -0,0 +1,99 @@
// Package keygen provides the AES key initializer function.
package keygen
import (
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"os"
"code.nonshy.com/nonshy/website/pkg/log"
)
// NewRSAKeys will generate an RSA 2048-bit key pair.
func NewRSAKeys() (*rsa.PrivateKey, error) {
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
return privateKey, err
}
// SerializePublicKey converts an RSA public key into an x509 PEM encoded byte string.
func SerializePublicKey(publicKey crypto.PublicKey) ([]byte, error) {
// Encode the public key to PEM format.
x509EncodedPub, err := x509.MarshalPKIXPublicKey(publicKey)
if err != nil {
return nil, err
}
pemEncodedPub := pem.EncodeToMemory(&pem.Block{
Type: "RSA PUBLIC KEY",
Bytes: x509EncodedPub,
})
return pemEncodedPub, nil
}
// DeserializePublicKey loads the RSA public key from the PEM encoded byte array.
func DeserializePublicKey(pemEncodedPub []byte) (*rsa.PublicKey, error) {
// Decode the public key.
log.Error("decode public key: %s", pemEncodedPub)
blockPub, _ := pem.Decode(pemEncodedPub)
x509EncodedPub := blockPub.Bytes
genericPublicKey, err := x509.ParsePKIXPublicKey(x509EncodedPub)
if err != nil {
return nil, err
}
publicKey := genericPublicKey.(*rsa.PublicKey)
return publicKey, nil
}
// WriteRSAKeys writes the public and private RSA keys to .pem files on disk.
func WriteRSAKeys(key *rsa.PrivateKey, privateFile, publicFile string) error {
// Encode the private key to PEM format.
x509Encoded := x509.MarshalPKCS1PrivateKey(key)
pemEncoded := pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509Encoded,
})
// Encode the public key to PEM format.
pemEncodedPub, err := SerializePublicKey(key.Public())
if err != nil {
return err
}
// Write the files.
if err := os.WriteFile(privateFile, pemEncoded, 0600); err != nil {
return err
}
if err := os.WriteFile(publicFile, pemEncodedPub, 0644); err != nil {
return err
}
return nil
}
// PrivateKeyFromFile loads the private key from disk.
func PrivateKeyFromFile(privateFile string) (*rsa.PrivateKey, error) {
// Read the private key file.
pemEncoded, err := os.ReadFile(privateFile)
if err != nil {
return nil, err
}
// Decode the private key.
block, _ := pem.Decode(pemEncoded)
x509Encoded := block.Bytes
privateKey, _ := x509.ParsePKCS1PrivateKey(x509Encoded)
return privateKey, nil
}
// PublicKeyFromFile loads the public key from disk.
func PublicKeyFromFile(publicFile string) (*rsa.PublicKey, error) {
pemEncodedPub, err := os.ReadFile(publicFile)
if err != nil {
return nil, err
}
// Decode the public key.
return DeserializePublicKey(pemEncodedPub)
}

View File

@ -9,6 +9,13 @@
</div>
</section>
<!-- Cold Storage warning -->
{{if .ColdStorageWarning}}
<div class="notification is-danger is-light mt-4">
<strong>Cold Storage Warning:</strong> {{.ColdStorageWarning}}.
</div>
{{end}}
<div class="block p-4">
<div class="columns">