Skip to content

Commit

Permalink
Merge pull request 2i2c-org#5291 from sunu/improve-nfs-restore-docs
Browse files Browse the repository at this point in the history 
Improve NFS home directory restore documentation
  • Loading branch information
@sgibson91
sgibson91 authored Jan 10, 2025
2 parents 5f04c39 + 0ae27fd commit 2f64673
Show file tree
Hide file tree
Showing 2 changed files with 139 additions and 20 deletions.
137 changes: 125 additions & 12 deletions deployer/commands/exec/infra_components.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,27 @@ def root_homes(
"--persist",
help="Do not automatically delete the pod after completing. Useful for long-running processes.",
),
extra_nfs_server: str = typer.Option(
None, help="IP address of an extra NFS server to mount"
additional_nfs_server: str = typer.Option(
None, help="IP address of an additional NFS server to mount"
),
extra_nfs_base_path: str = typer.Option(
None, help="Path of the extra NFS share to mount"
additional_nfs_base_path: str = typer.Option(
None, help="Path of the additional NFS share to mount"
),
extra_nfs_mount_path: str = typer.Option(
None, help="Mount point for the extra NFS share"
additional_nfs_mount_path: str = typer.Option(
None, help="Mount point for the additional NFS share"
),
additional_volume_id: str = typer.Option(
None,
help="ID of volume to mount (e.g. vol-1234567890abcdef0). "
"Only EBS volumes are supported for now.",
),
additional_volume_mount_path: str = typer.Option(
"/additional-volume", help="Mount point for the volume"
),
):
"""
Pop an interactive shell with the entire nfs file system of the given cluster mounted on /root-homes
Optionally mount an extra NFS share if required (useful when migrating data to a new NFS server).
Optionally mount an additional NFS or EBS volume if required (useful when migrating data to a new NFS server).
"""
config_file_path = find_absolute_path_to_cluster_file(cluster_name)
with open(config_file_path) as f:
Expand Down Expand Up @@ -82,17 +90,84 @@ def root_homes(
}
]

if extra_nfs_server and extra_nfs_base_path and extra_nfs_mount_path:
if additional_volume_id:
# Create PV for volume
pv_name = f"addtional-volume-{additional_volume_id}"
pv = {
"apiVersion": "v1",
"kind": "PersistentVolume",
"metadata": {"name": pv_name},
"spec": {
# We intentionally set the size to 1Mi to avoid having to
# specify and match the size of the underlying disk
# This doesn't trigger a resize of the underlying disk
# as long as the size matches the specified capacity
# of the PVC.
"capacity": {"storage": "1Mi"},
"volumeMode": "Filesystem",
"accessModes": ["ReadWriteOnce"],
"persistentVolumeReclaimPolicy": "Retain",
"storageClassName": "",
"csi": {
"driver": "ebs.csi.aws.com",
"fsType": "xfs",
"volumeHandle": additional_volume_id,
},
"mountOptions": [
"rw",
"relatime",
"nouuid",
"attr2",
"inode64",
"logbufs=8",
"logbsize=32k",
"pquota",
],
},
}

# Create PVC to bind to the PV
pvc = {
"apiVersion": "v1",
"kind": "PersistentVolumeClaim",
"metadata": {"name": pv_name, "namespace": hub_name},
"spec": {
"accessModes": ["ReadWriteOnce"],
"volumeName": pv_name,
"storageClassName": "",
# We intentionally set the size to 1Mi to avoid having to
# specify and match the size of the underlying disk
# This doesn't trigger a resize of the underlying disk
# as long as the size matches the specified capacity
# of the PV.
"resources": {"requests": {"storage": "1Mi"}},
},
}

volumes.append(
{
"name": "extra-nfs",
"nfs": {"server": extra_nfs_server, "path": extra_nfs_base_path},
"name": "additional-volume",
"persistentVolumeClaim": {"claimName": pv_name},
}
)
volume_mounts.append(
{"name": "additional-volume", "mountPath": additional_volume_mount_path}
)

if additional_nfs_server and additional_nfs_base_path and additional_nfs_mount_path:
volumes.append(
{
"name": "additional-nfs",
"nfs": {
"server": additional_nfs_server,
"path": additional_nfs_base_path,
},
}
)
volume_mounts.append(
{
"name": "extra-nfs",
"mountPath": extra_nfs_mount_path,
"name": "additional-nfs",
"mountPath": additional_nfs_mount_path,
}
)

Expand Down Expand Up @@ -141,6 +216,23 @@ def root_homes(

with cluster.auth():
try:
# If we have an additional volume to mount, create PV and PVC first
if additional_volume_id:
with tempfile.NamedTemporaryFile(
mode="w", suffix=".json"
) as pv_tmpf:
json.dump(pv, pv_tmpf)
pv_tmpf.flush()
subprocess.check_call(["kubectl", "create", "-f", pv_tmpf.name])

with tempfile.NamedTemporaryFile(
mode="w", suffix=".json"
) as pvc_tmpf:
json.dump(pvc, pvc_tmpf)
pvc_tmpf.flush()
subprocess.check_call(
["kubectl", "create", "-f", pvc_tmpf.name]
)
# We are splitting the previous `kubectl run` command into a
# create and exec couplet, because using run meant that the bash
# process we start would be assigned PID 1. This is a 'special'
Expand All @@ -152,11 +244,32 @@ def root_homes(
#
# Ask api-server to create a pod
subprocess.check_call(["kubectl", "create", "-f", tmpf.name])

# wait for the pod to be ready
print_colour("Waiting for the pod to be ready...", "yellow")
subprocess.check_call(
[
"kubectl",
"wait",
"-n",
hub_name,
f"pod/{pod_name}",
"--for=condition=Ready",
"--timeout=90s",
]
)

# Exec into pod
subprocess.check_call(exec_cmd)
finally:
if not persist:
delete_pod(pod_name, hub_name)
# Clean up PV and PVC if we created them
if additional_volume_id:
subprocess.check_call(
["kubectl", "-n", hub_name, "delete", "pvc", pv_name]
)
subprocess.check_call(["kubectl", "delete", "pv", pv_name])


@exec_app.command()
Expand Down
22 changes: 14 additions & 8 deletions docs/howto/filesystem-management/filesystem-backups/restore-filesystem.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,24 @@ To restore a home directory from a snapshot, we need to create a new EBS volume
Please follow AWS's guidance for [restoring EBS volumes from a snapshot](https://docs.aws.amazon.com/prescriptive-guidance/latest/backup-recovery/restore.html#restore-files) to create a new EBS volume from the snapshot.
```

Once we have created a new EBS volume from the snapshot, we need to mount it to the EC2 instance attached to the existing EBS volume containing the NFS home directories. To do this, we need to find the instance ID of the EC2 instance attached to the existing EBS volume. This involves the following steps:
Once we have created a new EBS volume from the snapshot, we can use the `deployer exec root-homes` command to mount the new EBS volume to a pod along with the existing NFS home directories volume.

1. Go to the EBS volumes page in the AWS console
2. Find the volume ID of the existing EBS volume containing the NFS home directories
3. Click on the volume ID and find the instance ID in the "Attached resources" section
4. Once we have the instance ID, we can mount the new EBS volume to the EC2 instance by following the steps outlined in the [Attaching an Amazon EBS volume to an instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-attaching-volume.html) guide.
```bash
deployer exec root-homes $CLUSTER_NAME $HUB_NAME --additional-volume-id=<new-ebs-volume-id> --additional-volume-mount-path=/restore-volume
```

Now, the NFS home directories volume is mounted to the pod along with the new EBS volume. We can now copy the contents from the restored EBS volume to the NFS home directories volume.

If we want to do a full restore and/or we are restoring data to a new and empty NFS home directories volume, we can use the following command to copy everything from the restored EBS volume to the NFS home directories volume.

```bash
rsync -av --info=progress2 /restore-volume/ /root-homes/
```

Once we have mounted the new EBS volume to the EC2 instance, we can copy the contents from the restored EBS volume to the NFS home directories volume. This can be done by running the following commands on the EC2 instance:
If we want to do a partial restore and we are restoring only a subset of the data to an existing NFS home directories volume, we can use the following command to copy only the necessary data from the restored EBS volume to the NFS home directories volume.

```bash
# Copy the contents from the restored EBS volume to the NFS home directories volume
rsync -av --info=progress2 </restored-volume/path-to-home-directories/> </existing-volume/path-to-home-directories/>
rsync -av --info=progress2 /restore-volume/<path-to-restore> /root-homes/<path-to-restore>
```

Once we have copied the contents from the restored EBS volume to the NFS home directories volume, we can delete the EBS volume that was created from the snapshot.
Expand Down

0 comments on commit 2f64673

Please sign in to comment.