dvc_minIO

The objective of this project is used dvc in ML and Data Science projects. For this we'll store large files in MinIO.

MinIO's Setup

1.Install Go sudo passwd root

su root

apt update

wget -c https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz

tar xvf go1.14.2.linux-amd64.tar.gz

sudo chown -R root:root ./go

sudo mv go /usr/local

sudo echo 'export PATH=$PATH:/usr/local/go/bin' >> /etc/profile

source /etc/profile

rm go1.14.2.linux-amd64.tar.gz

go version

2. Install MinIO server

cd ~

wget https://dl.min.io/server/minio/release/linux-amd64/minio

sudo useradd --system minio --shell /sbin/nologin

sudo usermod -L minio

sudo chage -E0 minio

sudo mv minio /usr/local/bin

sudo chmod +x /usr/local/bin/minio

sudo chown minio:minio /usr/local/bin/minio

sudo touch /etc/default/minio

sudo echo 'MINIO_ACCESS_KEY="minio"' >> /etc/default/minio

sudo echo 'MINIO_VOLUMES="/usr/local/share/minio/"' >> /etc/default/minio

sudo echo 'MINIO_OPTS="-C /etc/minio --address :9000"' >> /etc/default/minio

sudo echo 'MINIO_SECRET_KEY="gutteste"' >> /etc/default/minio

sudo mkdir /usr/local/share/minio

sudo mkdir /etc/minio

sudo chown minio:minio /usr/local/share/minio

sudo chown minio:minio /etc/minio

3. Install initialize script Minio's systemd

cd ~

wget https://raw.githubusercontent.com/minio/minio-service/master/linux-systemd/minio.service

vim minio_service

[delete all and paste the code available in /setup/minio_service.py]

sudo mv minio.service /etc/systemd/system

4.Firewall's configuration

cd ~

sudo ufw default deny incoming

sudo ufw default allow outgoing

sudo ufw allow ssh

sudo ufw allow 9000

sudo ufw allow http

sudo ufw allow https

sudo ufw enable

sudo ufw allow 80

sudo ufw allow 443

sudo ufw status verbose

5. Start Minio's service

cd ~ sudo systemctl daemon-reload

sudo systemctl enable minio

sudo systemctl start minio

sudo systemctl status minio

q (to quit)

6. Install Minio Customer (MC)

cd ~

wget https://dl.min.io/client/mc/release/linux-amd64/mc

chmod +x mc

7. Set Minio's service with S3 storage (Minio Customer)

cd ~ ./mc config host add minio http://127.0.0.1:9000 minio gutteste --api S3v4

./mc admin info minio

./mc mb minio/storagebr>

./mc ls

./mc share upload minio/storage/gutteste

8. Set Minio's storage with dvc

export MINIO_ACCESS_KEY="minio"

export MINIO_SECRET_KEY="gutteste"

Share: curl http://127.0.0.1:9000/storage/ -F x-amz-signature=57ed49f36625ef71333db75b122fa9c9ed583a87899f845df428624c02362ac2 -F bucket=storage -F policy=eyJleHBpcmF0aW9uIjoiMjAyMC0wOC0yOFQxNToyNTo1OS42ODZaIiwiY29uZGl0aW9ucyI6W1siZXEiLCIkYnVja2V0Iiwic3RvcmFnZSJdLFsiZXEiLCIka2V5IiwiZ3V0dGVzdGUiXSxbImVxIiwiJHgtYW16LWRhdGUiLCIyMDIwMDgyMVQxNTI1NTlaIl0sWyJlcSIsIiR4LWFtei1hbGdvcml0aG0iLCJBV1M0LUhNQUMtU0hBMjU2Il0sWyJlcSIsIiR4LWFtei1jcmVkZW50aWFsIiwibWluaW8vMjAyMDA4MjEvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJdXX0= -F x-amz-algorithm=AWS4-HMAC-SHA256 -F x-amz-credential=minio/20200821/us-east-1/s3/aws4_request -F x-amz-date=20200821T152559Z -F key=gutteste -F file=@

---

It can be important so I decided include here.

9. Create a hostname

cd ~

mkdir -m /etc/cloud/

cd /cloud

touch cloud.cfg

vim cloud.cfg

[Paste the code available in /setup/cloud.cfg.txt]

reboot

vim /etc/resolv.conf

[Paste the code available in /setup/resolv.conf.txt]

apt-get install bind9 bind9utils bind9-doc -y

cd /etc/bind/

cp named.conf.local named.conf.local.back

cp db.local db.fwd.gut.com

cp db.local db.rev.gut.com

mkdir /etc/bind/zones

mv db.fwd.gut.com zones

mv db.rev.gut.com zones

vim named.conf.local

[Paste the code available in /setup/name.config.local.txt]

cd zones

sudo vim db.fwd.gut.com

[Delete all and paste the code available in /setup/db.fwd.gut.com.txt]

vim db.rev.gut.com

[Delete all and paste the code available in /setup/db.rev.gut.com.txt]

service bind9 restart

10. Install and configure Apache

sudo apt install apache2

sudo ufw allow 'Apache'

11. Create Virtualhost

sudo mkdir /var/www/gut.com

sudo chown -R $USER:$USER /var/www/gut.com

sudo chmod -R 755 /var/www/gut.com

vim /var/www/gut.com/index.html

[Paste the code available in /setup/index.html]

sudo vim /etc/apache2/sites-available/gut.com.conf

[Paste the code available in /setup/gut.com.conf]

sudo mkdir -p /var/www/gut.com/documentroot

sudo a2ensite gut.com

systemctl reload apache2

sudo a2dissite 000-default.conf

systemctl reload apache2

vim /etc/apache2/apache2.conf

[Paste the code available in /setup/server_name.txt]

sudo systemctl restart apache2

12. Encrypt certificate using Certbot for MinIO

sudo apt-install software-properties-common

sudo add-apt-repository universe

sudo apt update

sudo apt install certbot

sudo service apache2 stop

sudo certbot certonly --standalone -d minio.gut.com

sudo cp /etc/letsencrypt/live/minio-server.your_domain_name/privkey.pem /etc/minio/certs/private.key

sudo cp /etc/letsencrypt/live/minio-server.your_domain_name/fullchain.pem /etc/minio/certs/public.crt

sudo chown minio:minio /etc/minio/certs/private.key

sudo chown minio:minio /etc/minio/certs/public.crt

sudo systemctl restart minio

13. Protecing access with TLS certificate (http - https)

cd ~ mkdir -p /etc/ssl/minio/ssl.crt/

mkdir -p /etc/ssl/minio/ssl.key/

cd /etc/ssl/minio/

touch openssl.conf

vim touch openssl.conf

[Paste the code available in /setup/openssl.conf.txt]

openssl req -x509 -nodes -days 730 -newkey rsa:2048 -keyout private.key -out public.crt -config openssl.conf

cp private.key /root/.minio/certs/CAs/

cp public.crt /root/.minio/certs/CAs/

mv private.key /etc/ssl/minio/ssl.key/

mv public.crt /etc/ssl/minio/ssl.crt/

14.Adjustment virtualhost config to include the certificates

cd ~

vim /etc/apache2/sites-available/default-ssl.conf

[Delete all, copie and paste the code available in /setup/ssl.certicates.txt]

a2ensite default-ssl.conf

systemctl reload apache2