Skip to content

mag_minio

mag_minio

alias()

-a, --alias

Source code in mag_minio/minio_options.py 
14
15
16
17
18
19
def alias():
  """-a, --alias"""
  return click.option('-a', '--alias', default="myminio", 
                     show_default=True, 
                     help='alias name. To see the list run `mc alias list`.', 
                     callback=validate_tenant_callback)

api(realm, tenant, ports)

Launch minio server API. Launch this to be able to use mc command.

Source code in mag_minio/minio.py 
78
79
80
81
82
83
84
85
@minio.command
@options.realm
@options.ports(default="9000:9000")
@minio_options.tenant()
def api(realm, tenant, ports):
  """Launch minio server API. Launch this to be able to use mc command."""
  click.echo("")
  launch_hl(realm=realm, tenant=tenant, ports=ports, wait=True)

bucket(realm, tenant, alias, ports, bucket_name)

Create a minio bucket in the tenant

Source code in mag_minio/minio.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
@add.command(aliases=["b"])
@options.realm
@options.ports(default="9000:9000")
@minio_options.tenant()
@minio_options.alias()
@click.option("-b", "--bucket-name", help="Bucket name. If not set will create magasin-?????", default="")
def bucket(realm, tenant, alias, ports, bucket_name):
  """Create a minio bucket in the tenant"""
  click.echo("Create Bucket")  

  # TODO use https://min.io/docs/minio/linux/developers/python/minio-py.html
  # May need Minio(..., cert_check=False)
  launch_hl(realm=realm, tenant=tenant, ports=ports)
  from time import sleep 
  sleep(1)
  # Check if the tenant alias is set
  if not mc.check_mc_admin_info(alias):
    # TODO This needs to be removed if bucket is created with python library
    click.echo(f"minio tenant configuration alias '{alias}' not set. Try running:", err=True)
    click.echo(f"      mag minio api; mc alias set {alias} http://localhost:9000 <accesskey/user> <secretkey/password> --insecure")
    exit(-1)
  else: 
    click.echo("alias check successful")
  if bucket_name == "":
      bucket_name = "magasin-" + generate_random_string(5)
  try:
    mc_command = f"mc mb {alias}/{bucket_name} --insecure"
    click.echo("mc command:" + mc_command)    
    subprocess.run(mc_command, shell=True)
  except Exception as e:
    print('exception', e)

cli(ctx, verbose)

magasin client is the glue between magasin components, it makes easier common tasks

Source code in mag/mag.py 
39
40
41
42
43
44
45
46
@click.group(cls=ClickAliasedGroup)
@click.option('-v', '--verbose', count=True)
@click.option('--version', is_flag=True, callback=print_version,
              expose_value=False, is_eager=True)
@click.pass_context
def cli(ctx, verbose):
    """magasin client is the glue between magasin components, it makes easier common tasks"""
    ctx.ensure_object(dict)

forward_port(realm, component, service_name, ports, verbose=False)

Forward ports for the specified realm, component, and service name.

Parameters:

Name Type Description Default
realm str

A string representing the realm.

 required
component str

A string representing the component.

 required
service_name str

(str) A string representing the service name. The service name can be obtained using kubectl get services --namespace magasin-superset).

 required
ports str

A string representing the ports to be forwarded (example: "8000:8000").

 required
verbose bool

A boolean indicating whether to enable verbose mode (default is False).

 False

Returns: None

Usage:

forward_port(realm, component, service_name, ports, verbose)

Example: 

# Given this
kubectl get service -n magasin-superset
NAME                      TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
superset                  ClusterIP   10.100.96.47     <none>        8088/TCP   7d22h
You can forward this service 
forward_port("magasin", "superset", "superset", "8088:8088")

Notes:

  • Assumes the port_forward_command function is defined elsewhere in the code.
  • Uses subprocess.Popen to launch the port forwarding command in a subprocess.
  • Registers the terminate_process function using atexit.register, ensuring that the port forwarding process is terminated when the script exits.
Source code in mag/mag_core/launch.py 
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
def forward_port(realm: str, component: str, service_name: str, ports: str, verbose=False) -> None:
    """
    Forward ports for the specified realm, component, and service name.

    Args:
        realm (str): A string representing the realm.
        component (str): A string representing the component.
        service_name: (str) A string representing the service name. The service name can be obtained using kubectl get services --namespace magasin-superset).
        ports (str): A string representing the ports to be forwarded (example: "8000:8000").
        verbose (bool): A boolean indicating whether to enable verbose mode (default is False).

    Returns:
    None

    Usage:

    forward_port(realm, component, service_name, ports, verbose)

    Example:
    ```
    # Given this
    kubectl get service -n magasin-superset
    NAME                      TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
    superset                  ClusterIP   10.100.96.47     <none>        8088/TCP   7d22h
    ```
    You can forward this service
    ```
    forward_port("magasin", "superset", "superset", "8088:8088")
    ```

    Notes:

    - Assumes the port_forward_command function is defined elsewhere in the code.
    - Uses subprocess.Popen to launch the port forwarding command in a subprocess.
    - Registers the terminate_process function using atexit.register, ensuring that the port forwarding process
      is terminated when the script exits.
    """
    port_forward_command = port_forward_command_arr(
        realm, component, service_name, ports, verbose)
    click.echo("forward_port command: " + " ".join(port_forward_command))
    process = subprocess.Popen(port_forward_command, shell=False)

    local, _ = split_ports(ports)
    click.echo("Waiting for port to be open...")
    if not is_port_open(host='localhost', port=local):
        click.echo("Port could not be opened.")
        exit(-1)
    click.echo("Port ready.")

    atexit.register(terminate_process, process)

generate_random_string(length=7)

Generate a random alphanumeric lowercase string of a specified length.

Parameters: - length (int): The desired length of the random string.

Returns: - str: A random string containing letters (both lowercase and uppercase) and digits.

Source code in mag/mag_core/random.py 
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
def generate_random_string(length=7):
    """
    Generate a random alphanumeric lowercase string of a specified length.

    Parameters:
    - length (int): The desired length of the random string.

    Returns:
    - str: A random string containing letters (both lowercase and uppercase) and digits.
    """
    characters = string.ascii_lowercase + string.digits
    return ''.join(random.choice(characters) for _ in range(length))

launch_ui(realm, component, service_name, ports, protocol='http', verbose=False)

Launches the user interface for a given realm, component, and service.

Parameters:

Name Type Description Default
realm str

The realm of the magasin instance.

 required
component str

The magasin component (f.i superset, daskhub, drill, ...)

 required
service_name str

The name of the kubernetes service to forward.

 required
ports str

The ports to forward, using the format "local_port:remote_port".

 required
protocol str

The protocol to use (default is "http").

 'http'
verbose bool

Whether to display verbose output (default is False).

 False

Returns:

Name Type Description
None None

Nothing
Source code in mag/mag_core/launch.py 
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
def launch_ui(realm: str, component: str, service_name: str, ports: str, protocol: str = "http", verbose=False) -> None:
    """
    Launches the user interface for a given realm, component, and service.

    Args:
        realm (str): The realm of the magasin instance.
        component (str): The magasin component (f.i superset, daskhub, drill, ...)
        service_name (str): The name of the kubernetes service to forward.
        ports (str): The ports to forward, using the format "local_port:remote_port".
        protocol (str, optional): The protocol to use (default is "http").
        verbose (bool, optional): Whether to display verbose output (default is False).

    Returns:
        None: Nothing
    """    
    forward_port(realm=realm, component=component,
                 service_name=service_name, ports=ports, verbose=verbose)

    localhost_port, _ = split_ports(ports)
    url = f"{protocol}://localhost:{localhost_port}"
    click.echo(f"Open browser at: {url}")
    click.launch(url)
    click.echo("launch ui")

    try:
        # Wait for user to press Ctrl+C
        signal.pause()
    except KeyboardInterrupt:
        # Handle Ctrl+C: terminate the server and clean up
        process.terminate()
        os.waitpid(process.pid, 0)
        click.echo("\nServer terminated. Exiting.")

minio(realm, tenant)

MinIO commands

Source code in mag_minio/minio.py 
34
35
36
37
38
@cli.group('minio', cls=ClickAliasedGroup, aliases=["m"])
@options.realm
@minio_options.tenant()
def minio(realm, tenant):
  """MinIO commands"""

sui(realm, tenant, ports)

Launch user tenant ssl secured user interface

Source code in mag_minio/minio.py 
47
48
49
50
51
52
53
@minio.command
@options.realm
@options.ports(default="9443:9443")
@minio_options.tenant()
def sui(realm, tenant, ports):
  """Launch user tenant ssl secured user interface"""
  launch_ui(realm, component=COMPONENT, service_name=f"svc/{tenant}-console", ports=ports, protocol="https")

tenant()

-t, --tenant

Source code in mag_minio/minio_options.py 
 7
 8
 9
10
11
12
def tenant():
  """-t, --tenant"""
  return click.option('-t', '--tenant', default="myminio", 
                     show_default=True, 
                     help='minio tenant name.', 
                     callback=validate_tenant_callback)

ui(realm, tenant, ports)

Launch user tenant non-ssl user interface

Source code in mag_minio/minio.py 
59
60
61
62
63
64
65
@minio.command
@options.realm
@options.ports(default="9090:9090")
@minio_options.tenant()
def ui(realm, tenant, ports):
  """Launch user tenant non-ssl user interface"""
  launch_ui(realm, component=COMPONENT, service_name=f"svc/{tenant}-console", ports=ports, protocol="http")

user(realm, tenant, ports, accesskey, secretkey)

Create a minio user in the tenant

Source code in mag_minio/minio.py 
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
@add.command(aliases=["u"])
@options.realm
@options.ports(default="9000:9000")
@minio_options.tenant()
@click.option("-a", "--accesskey", help="user name", default="")
@click.option("-s", "--secretkey", help="password", default="")
def user(realm, tenant, ports, accesskey, secretkey):
  """Create a minio user in the tenant"""

  # TODO use https://min.io/docs/minio/linux/developers/python/minio-py.html
  # May need Minio(..., cert_check=False)
  launch_hl(realm=realm, tenant=tenant, ports=ports)
  from time import sleep 
  sleep(1)
  # Check if the tenant alias is set
  if not mc.check_mc_admin_info(tenant):
    # TODO This needs to be removed if bucket is created with python library
    click.echo(f"minio tenant configuration alias '{tenant}' not set. Try running:", err=True)
    click.echo(f"      mag minio api; mc alias set {tenant} https://localhost:9000 <accesskey/user> <secretkey/password> --insecure")
    exit(-1)
  else: 
     click.echo(f"minio tenant ok")

  if accesskey == "":
      accesskey = "magasin-" + generate_random_string(5)
  if secretkey == "":
      secretkey = generate_random_string(32)
  try:
    mc_command = f"mc admin user add {tenant} {accesskey} {secretkey} --insecure"
    click.echo("mc command:" + mc_command)    
    subprocess.run(mc_command, shell=True)
    click.echo("")
    click.echo(f"Created\n    user/accesskey: {accesskey}\n    password/secretkey: {secretkey}")
    click.echo(f"\nNote: you still need to assign permissions/policies to the user. You can do it with the UI: `mag minio ui`")
    click.echo("")
  except Exception as e:
    print('exception', e)