aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/scsi/bfa
diff options
context:
space:
mode:
authorJing Huang <huangj@brocade.com>2010-03-19 14:05:39 -0400
committerJames Bottomley <James.Bottomley@suse.de>2010-04-11 10:24:16 -0400
commitb504293fe9dc42917a919044f2b672fb361329d0 (patch)
tree13a7a576560155c6c8b49976e286060b5c904aba /drivers/scsi/bfa
parent12fb8c1574d7d0c262d2f4c667047889c4f27ebe (diff)
[SCSI] bfa: add fc transport class based vport create/delete
Use duplicate fc transport template for physical and vitual port. Add vport create/delete/disalbe functions in the transport template of physical port. Changes to make the vport create/delete function to work under this framework. Signed-off-by: Jing Huang <huangj@brocade.com> Signed-off-by: James Bottomley <James.Bottomley@suse.de>
Diffstat (limited to 'drivers/scsi/bfa')
-rw-r--r--drivers/scsi/bfa/bfad.c10
-rw-r--r--drivers/scsi/bfa/bfad_attr.c201
-rw-r--r--drivers/scsi/bfa/bfad_drv.h2
-rw-r--r--drivers/scsi/bfa/bfad_im.c53
-rw-r--r--drivers/scsi/bfa/bfad_im.h6
5 files changed, 239 insertions, 33 deletions
diff --git a/drivers/scsi/bfa/bfad.c b/drivers/scsi/bfa/bfad.c
index 13f5feb308c2..3a5163d3675d 100644
--- a/drivers/scsi/bfa/bfad.c
+++ b/drivers/scsi/bfa/bfad.c
@@ -299,8 +299,6 @@ bfa_fcb_vport_delete(struct bfad_vport_s *vport_drv)
299 complete(vport_drv->comp_del); 299 complete(vport_drv->comp_del);
300 return; 300 return;
301 } 301 }
302
303 kfree(vport_drv);
304} 302}
305 303
306/** 304/**
@@ -483,7 +481,7 @@ ext:
483 */ 481 */
484bfa_status_t 482bfa_status_t
485bfad_vport_create(struct bfad_s *bfad, u16 vf_id, 483bfad_vport_create(struct bfad_s *bfad, u16 vf_id,
486 struct bfa_port_cfg_s *port_cfg) 484 struct bfa_port_cfg_s *port_cfg, struct device *dev)
487{ 485{
488 struct bfad_vport_s *vport; 486 struct bfad_vport_s *vport;
489 int rc = BFA_STATUS_OK; 487 int rc = BFA_STATUS_OK;
@@ -506,7 +504,8 @@ bfad_vport_create(struct bfad_s *bfad, u16 vf_id,
506 goto ext_free_vport; 504 goto ext_free_vport;
507 505
508 if (port_cfg->roles & BFA_PORT_ROLE_FCP_IM) { 506 if (port_cfg->roles & BFA_PORT_ROLE_FCP_IM) {
509 rc = bfad_im_scsi_host_alloc(bfad, vport->drv_port.im_port); 507 rc = bfad_im_scsi_host_alloc(bfad, vport->drv_port.im_port,
508 dev);
510 if (rc != BFA_STATUS_OK) 509 if (rc != BFA_STATUS_OK)
511 goto ext_free_fcs_vport; 510 goto ext_free_fcs_vport;
512 } 511 }
@@ -848,7 +847,8 @@ bfad_cfg_pport(struct bfad_s *bfad, enum bfa_port_role role)
848 goto out; 847 goto out;
849 } 848 }
850 849
851 rc = bfad_im_scsi_host_alloc(bfad, bfad->pport.im_port); 850 rc = bfad_im_scsi_host_alloc(bfad, bfad->pport.im_port,
851 &bfad->pcidev->dev);
852 if (rc != BFA_STATUS_OK) 852 if (rc != BFA_STATUS_OK)
853 goto out; 853 goto out;
854 854
diff --git a/drivers/scsi/bfa/bfad_attr.c b/drivers/scsi/bfa/bfad_attr.c
index 6a2efdd5ef24..e477bfbfa7d8 100644
--- a/drivers/scsi/bfa/bfad_attr.c
+++ b/drivers/scsi/bfa/bfad_attr.c
@@ -364,6 +364,152 @@ bfad_im_set_rport_loss_tmo(struct fc_rport *rport, u32 timeout)
364 364
365} 365}
366 366
367static int
368bfad_im_vport_create(struct fc_vport *fc_vport, bool disable)
369{
370 char *vname = fc_vport->symbolic_name;
371 struct Scsi_Host *shost = fc_vport->shost;
372 struct bfad_im_port_s *im_port =
373 (struct bfad_im_port_s *) shost->hostdata[0];
374 struct bfad_s *bfad = im_port->bfad;
375 struct bfa_port_cfg_s port_cfg;
376 int status = 0, rc;
377 unsigned long flags;
378
379 memset(&port_cfg, 0, sizeof(port_cfg));
380
381 port_cfg.pwwn = wwn_to_u64((u8 *) &fc_vport->port_name);
382 port_cfg.nwwn = wwn_to_u64((u8 *) &fc_vport->node_name);
383
384 if (strlen(vname) > 0)
385 strcpy((char *)&port_cfg.sym_name, vname);
386
387 port_cfg.roles = BFA_PORT_ROLE_FCP_IM;
388 rc = bfad_vport_create(bfad, 0, &port_cfg, &fc_vport->dev);
389
390 if (rc == BFA_STATUS_OK) {
391 struct bfad_vport_s *vport;
392 struct bfa_fcs_vport_s *fcs_vport;
393 struct Scsi_Host *vshost;
394
395 spin_lock_irqsave(&bfad->bfad_lock, flags);
396 fcs_vport = bfa_fcs_vport_lookup(&bfad->bfa_fcs, 0,
397 port_cfg.pwwn);
398 if (fcs_vport == NULL) {
399 spin_unlock_irqrestore(&bfad->bfad_lock, flags);
400 return VPCERR_BAD_WWN;
401 }
402
403 fc_vport_set_state(fc_vport, FC_VPORT_ACTIVE);
404 if (disable) {
405 bfa_fcs_vport_stop(fcs_vport);
406 fc_vport_set_state(fc_vport, FC_VPORT_DISABLED);
407 }
408 spin_unlock_irqrestore(&bfad->bfad_lock, flags);
409
410 vport = fcs_vport->vport_drv;
411 vshost = vport->drv_port.im_port->shost;
412 fc_host_node_name(vshost) = wwn_to_u64((u8 *) &port_cfg.nwwn);
413 fc_host_port_name(vshost) = wwn_to_u64((u8 *) &port_cfg.pwwn);
414 fc_vport->dd_data = vport;
415 vport->drv_port.im_port->fc_vport = fc_vport;
416
417 } else if (rc == BFA_STATUS_INVALID_WWN)
418 return VPCERR_BAD_WWN;
419 else if (rc == BFA_STATUS_VPORT_EXISTS)
420 return VPCERR_BAD_WWN;
421 else if (rc == BFA_STATUS_VPORT_MAX)
422 return VPCERR_NO_FABRIC_SUPP;
423 else if (rc == BFA_STATUS_VPORT_WWN_BP)
424 return VPCERR_BAD_WWN;
425 else
426 return FC_VPORT_FAILED;
427
428 return status;
429}
430
431static int
432bfad_im_vport_delete(struct fc_vport *fc_vport)
433{
434 struct bfad_vport_s *vport = (struct bfad_vport_s *)fc_vport->dd_data;
435 struct bfad_im_port_s *im_port =
436 (struct bfad_im_port_s *) vport->drv_port.im_port;
437 struct bfad_s *bfad = im_port->bfad;
438 struct bfad_port_s *port;
439 struct bfa_fcs_vport_s *fcs_vport;
440 struct Scsi_Host *vshost;
441 wwn_t pwwn;
442 int rc;
443 unsigned long flags;
444 struct completion fcomp;
445
446 if (im_port->flags & BFAD_PORT_DELETE)
447 goto free_scsi_host;
448
449 port = im_port->port;
450
451 vshost = vport->drv_port.im_port->shost;
452 pwwn = wwn_to_u64((u8 *) &fc_host_port_name(vshost));
453
454 spin_lock_irqsave(&bfad->bfad_lock, flags);
455 fcs_vport = bfa_fcs_vport_lookup(&bfad->bfa_fcs, 0, pwwn);
456 spin_unlock_irqrestore(&bfad->bfad_lock, flags);
457
458 if (fcs_vport == NULL)
459 return VPCERR_BAD_WWN;
460
461 vport->drv_port.flags |= BFAD_PORT_DELETE;
462
463 vport->comp_del = &fcomp;
464 init_completion(vport->comp_del);
465
466 spin_lock_irqsave(&bfad->bfad_lock, flags);
467 rc = bfa_fcs_vport_delete(&vport->fcs_vport);
468 spin_unlock_irqrestore(&bfad->bfad_lock, flags);
469
470 wait_for_completion(vport->comp_del);
471
472free_scsi_host:
473 bfad_os_scsi_host_free(bfad, im_port);
474
475 kfree(vport);
476
477 return 0;
478}
479
480static int
481bfad_im_vport_disable(struct fc_vport *fc_vport, bool disable)
482{
483 struct bfad_vport_s *vport;
484 struct bfad_s *bfad;
485 struct bfa_fcs_vport_s *fcs_vport;
486 struct Scsi_Host *vshost;
487 wwn_t pwwn;
488 unsigned long flags;
489
490 vport = (struct bfad_vport_s *)fc_vport->dd_data;
491 bfad = vport->drv_port.bfad;
492 vshost = vport->drv_port.im_port->shost;
493 pwwn = wwn_to_u64((u8 *) &fc_vport->port_name);
494
495 spin_lock_irqsave(&bfad->bfad_lock, flags);
496 fcs_vport = bfa_fcs_vport_lookup(&bfad->bfa_fcs, 0, pwwn);
497 spin_unlock_irqrestore(&bfad->bfad_lock, flags);
498
499 if (fcs_vport == NULL)
500 return VPCERR_BAD_WWN;
501
502 if (disable) {
503 bfa_fcs_vport_stop(fcs_vport);
504 fc_vport_set_state(fc_vport, FC_VPORT_DISABLED);
505 } else {
506 bfa_fcs_vport_start(fcs_vport);
507 fc_vport_set_state(fc_vport, FC_VPORT_ACTIVE);
508 }
509
510 return 0;
511}
512
367struct fc_function_template bfad_im_fc_function_template = { 513struct fc_function_template bfad_im_fc_function_template = {
368 514
369 /* Target dynamic attributes */ 515 /* Target dynamic attributes */
@@ -413,6 +559,61 @@ struct fc_function_template bfad_im_fc_function_template = {
413 .show_rport_dev_loss_tmo = 1, 559 .show_rport_dev_loss_tmo = 1,
414 .get_rport_dev_loss_tmo = bfad_im_get_rport_loss_tmo, 560 .get_rport_dev_loss_tmo = bfad_im_get_rport_loss_tmo,
415 .set_rport_dev_loss_tmo = bfad_im_set_rport_loss_tmo, 561 .set_rport_dev_loss_tmo = bfad_im_set_rport_loss_tmo,
562
563 .vport_create = bfad_im_vport_create,
564 .vport_delete = bfad_im_vport_delete,
565 .vport_disable = bfad_im_vport_disable,
566};
567
568struct fc_function_template bfad_im_vport_fc_function_template = {
569
570 /* Target dynamic attributes */
571 .get_starget_port_id = bfad_im_get_starget_port_id,
572 .show_starget_port_id = 1,
573 .get_starget_node_name = bfad_im_get_starget_node_name,
574 .show_starget_node_name = 1,
575 .get_starget_port_name = bfad_im_get_starget_port_name,
576 .show_starget_port_name = 1,
577
578 /* Host dynamic attribute */
579 .get_host_port_id = bfad_im_get_host_port_id,
580 .show_host_port_id = 1,
581
582 /* Host fixed attributes */
583 .show_host_node_name = 1,
584 .show_host_port_name = 1,
585 .show_host_supported_classes = 1,
586 .show_host_supported_fc4s = 1,
587 .show_host_supported_speeds = 1,
588 .show_host_maxframe_size = 1,
589
590 /* More host dynamic attributes */
591 .show_host_port_type = 1,
592 .get_host_port_type = bfad_im_get_host_port_type,
593 .show_host_port_state = 1,
594 .get_host_port_state = bfad_im_get_host_port_state,
595 .show_host_active_fc4s = 1,
596 .get_host_active_fc4s = bfad_im_get_host_active_fc4s,
597 .show_host_speed = 1,
598 .get_host_speed = bfad_im_get_host_speed,
599 .show_host_fabric_name = 1,
600 .get_host_fabric_name = bfad_im_get_host_fabric_name,
601
602 .show_host_symbolic_name = 1,
603
604 /* Statistics */
605 .get_fc_host_stats = bfad_im_get_stats,
606 .reset_fc_host_stats = bfad_im_reset_stats,
607
608 /* Allocation length for host specific data */
609 .dd_fcrport_size = sizeof(struct bfad_itnim_data_s *),
610
611 /* Remote port fixed attributes */
612 .show_rport_maxframe_size = 1,
613 .show_rport_supported_classes = 1,
614 .show_rport_dev_loss_tmo = 1,
615 .get_rport_dev_loss_tmo = bfad_im_get_rport_loss_tmo,
616 .set_rport_dev_loss_tmo = bfad_im_set_rport_loss_tmo,
416}; 617};
417 618
418/** 619/**
diff --git a/drivers/scsi/bfa/bfad_drv.h b/drivers/scsi/bfa/bfad_drv.h
index 107848cd3b6d..f4b14396c4ac 100644
--- a/drivers/scsi/bfa/bfad_drv.h
+++ b/drivers/scsi/bfa/bfad_drv.h
@@ -254,7 +254,7 @@ do { \
254 254
255 255
256bfa_status_t bfad_vport_create(struct bfad_s *bfad, u16 vf_id, 256bfa_status_t bfad_vport_create(struct bfad_s *bfad, u16 vf_id,
257 struct bfa_port_cfg_s *port_cfg); 257 struct bfa_port_cfg_s *port_cfg, struct device *dev);
258bfa_status_t bfad_vf_create(struct bfad_s *bfad, u16 vf_id, 258bfa_status_t bfad_vf_create(struct bfad_s *bfad, u16 vf_id,
259 struct bfa_port_cfg_s *port_cfg); 259 struct bfa_port_cfg_s *port_cfg);
260bfa_status_t bfad_cfg_pport(struct bfad_s *bfad, enum bfa_port_role role); 260bfa_status_t bfad_cfg_pport(struct bfad_s *bfad, enum bfa_port_role role);
diff --git a/drivers/scsi/bfa/bfad_im.c b/drivers/scsi/bfa/bfad_im.c
index e2c70de2dba5..f263891b8cc6 100644
--- a/drivers/scsi/bfa/bfad_im.c
+++ b/drivers/scsi/bfa/bfad_im.c
@@ -30,6 +30,7 @@ BFA_TRC_FILE(LDRV, IM);
30 30
31DEFINE_IDR(bfad_im_port_index); 31DEFINE_IDR(bfad_im_port_index);
32struct scsi_transport_template *bfad_im_scsi_transport_template; 32struct scsi_transport_template *bfad_im_scsi_transport_template;
33struct scsi_transport_template *bfad_im_scsi_vport_transport_template;
33static void bfad_im_itnim_work_handler(struct work_struct *work); 34static void bfad_im_itnim_work_handler(struct work_struct *work);
34static int bfad_im_queuecommand(struct scsi_cmnd *cmnd, 35static int bfad_im_queuecommand(struct scsi_cmnd *cmnd,
35 void (*done)(struct scsi_cmnd *)); 36 void (*done)(struct scsi_cmnd *));
@@ -512,7 +513,8 @@ void bfa_fcb_itnim_tov(struct bfad_itnim_s *itnim)
512 * Allocate a Scsi_Host for a port. 513 * Allocate a Scsi_Host for a port.
513 */ 514 */
514int 515int
515bfad_im_scsi_host_alloc(struct bfad_s *bfad, struct bfad_im_port_s *im_port) 516bfad_im_scsi_host_alloc(struct bfad_s *bfad, struct bfad_im_port_s *im_port,
517 struct device *dev)
516{ 518{
517 int error = 1; 519 int error = 1;
518 520
@@ -541,12 +543,15 @@ bfad_im_scsi_host_alloc(struct bfad_s *bfad, struct bfad_im_port_s *im_port)
541 im_port->shost->max_lun = MAX_FCP_LUN; 543 im_port->shost->max_lun = MAX_FCP_LUN;
542 im_port->shost->max_cmd_len = 16; 544 im_port->shost->max_cmd_len = 16;
543 im_port->shost->can_queue = bfad->cfg_data.ioc_queue_depth; 545 im_port->shost->can_queue = bfad->cfg_data.ioc_queue_depth;
544 im_port->shost->transportt = bfad_im_scsi_transport_template; 546 if (im_port->port->pvb_type == BFAD_PORT_PHYS_BASE)
547 im_port->shost->transportt = bfad_im_scsi_transport_template;
548 else
549 im_port->shost->transportt =
550 bfad_im_scsi_vport_transport_template;
545 551
546 error = bfad_os_scsi_add_host(im_port->shost, im_port, bfad); 552 error = scsi_add_host(im_port->shost, dev);
547 if (error) { 553 if (error) {
548 printk(KERN_WARNING "bfad_os_scsi_add_host failure %d\n", 554 printk(KERN_WARNING "scsi_add_host failure %d\n", error);
549 error);
550 goto out_fc_rel; 555 goto out_fc_rel;
551 } 556 }
552 557
@@ -588,9 +593,11 @@ bfad_im_port_delete_handler(struct work_struct *work)
588 struct bfad_im_port_s *im_port = 593 struct bfad_im_port_s *im_port =
589 container_of(work, struct bfad_im_port_s, port_delete_work); 594 container_of(work, struct bfad_im_port_s, port_delete_work);
590 595
591 bfad_im_scsi_host_free(im_port->bfad, im_port); 596 if (im_port->port->pvb_type != BFAD_PORT_PHYS_BASE) {
592 bfad_im_port_clean(im_port); 597 im_port->flags |= BFAD_PORT_DELETE;
593 kfree(im_port); 598 fc_vport_terminate(im_port->fc_vport);
599 }
600
594} 601}
595 602
596bfa_status_t 603bfa_status_t
@@ -689,23 +696,6 @@ bfad_im_probe_undo(struct bfad_s *bfad)
689 } 696 }
690} 697}
691 698
692
693
694
695int
696bfad_os_scsi_add_host(struct Scsi_Host *shost, struct bfad_im_port_s *im_port,
697 struct bfad_s *bfad)
698{
699 struct device *dev;
700
701 if (im_port->port->pvb_type == BFAD_PORT_PHYS_BASE)
702 dev = &bfad->pcidev->dev;
703 else
704 dev = &bfad->pport.im_port->shost->shost_gendev;
705
706 return scsi_add_host(shost, dev);
707}
708
709struct Scsi_Host * 699struct Scsi_Host *
710bfad_os_scsi_host_alloc(struct bfad_im_port_s *im_port, struct bfad_s *bfad) 700bfad_os_scsi_host_alloc(struct bfad_im_port_s *im_port, struct bfad_s *bfad)
711{ 701{
@@ -724,7 +714,8 @@ bfad_os_scsi_host_alloc(struct bfad_im_port_s *im_port, struct bfad_s *bfad)
724void 714void
725bfad_os_scsi_host_free(struct bfad_s *bfad, struct bfad_im_port_s *im_port) 715bfad_os_scsi_host_free(struct bfad_s *bfad, struct bfad_im_port_s *im_port)
726{ 716{
727 flush_workqueue(bfad->im->drv_workq); 717 if (!(im_port->flags & BFAD_PORT_DELETE))
718 flush_workqueue(bfad->im->drv_workq);
728 bfad_im_scsi_host_free(im_port->bfad, im_port); 719 bfad_im_scsi_host_free(im_port->bfad, im_port);
729 bfad_im_port_clean(im_port); 720 bfad_im_port_clean(im_port);
730 kfree(im_port); 721 kfree(im_port);
@@ -829,6 +820,13 @@ bfad_im_module_init(void)
829 if (!bfad_im_scsi_transport_template) 820 if (!bfad_im_scsi_transport_template)
830 return BFA_STATUS_ENOMEM; 821 return BFA_STATUS_ENOMEM;
831 822
823 bfad_im_scsi_vport_transport_template =
824 fc_attach_transport(&bfad_im_vport_fc_function_template);
825 if (!bfad_im_scsi_vport_transport_template) {
826 fc_release_transport(bfad_im_scsi_transport_template);
827 return BFA_STATUS_ENOMEM;
828 }
829
832 return BFA_STATUS_OK; 830 return BFA_STATUS_OK;
833} 831}
834 832
@@ -837,6 +835,8 @@ bfad_im_module_exit(void)
837{ 835{
838 if (bfad_im_scsi_transport_template) 836 if (bfad_im_scsi_transport_template)
839 fc_release_transport(bfad_im_scsi_transport_template); 837 fc_release_transport(bfad_im_scsi_transport_template);
838 if (bfad_im_scsi_vport_transport_template)
839 fc_release_transport(bfad_im_scsi_vport_transport_template);
840} 840}
841 841
842void 842void
@@ -937,6 +937,7 @@ bfad_os_fc_host_init(struct bfad_im_port_s *im_port)
937 bfa_os_htonll((bfa_fcs_port_get_nwwn(port->fcs_port))); 937 bfa_os_htonll((bfa_fcs_port_get_nwwn(port->fcs_port)));
938 fc_host_port_name(host) = 938 fc_host_port_name(host) =
939 bfa_os_htonll((bfa_fcs_port_get_pwwn(port->fcs_port))); 939 bfa_os_htonll((bfa_fcs_port_get_pwwn(port->fcs_port)));
940 fc_host_max_npiv_vports(host) = bfa_lps_get_max_vport(&bfad->bfa);
940 941
941 fc_host_supported_classes(host) = FC_COS_CLASS3; 942 fc_host_supported_classes(host) = FC_COS_CLASS3;
942 943
diff --git a/drivers/scsi/bfa/bfad_im.h b/drivers/scsi/bfa/bfad_im.h
index 85ab2da21321..973cab4d09c7 100644
--- a/drivers/scsi/bfa/bfad_im.h
+++ b/drivers/scsi/bfa/bfad_im.h
@@ -34,7 +34,7 @@ void bfad_im_port_online(struct bfad_s *bfad, struct bfad_port_s *port);
34void bfad_im_port_offline(struct bfad_s *bfad, struct bfad_port_s *port); 34void bfad_im_port_offline(struct bfad_s *bfad, struct bfad_port_s *port);
35void bfad_im_port_clean(struct bfad_im_port_s *im_port); 35void bfad_im_port_clean(struct bfad_im_port_s *im_port);
36int bfad_im_scsi_host_alloc(struct bfad_s *bfad, 36int bfad_im_scsi_host_alloc(struct bfad_s *bfad,
37 struct bfad_im_port_s *im_port); 37 struct bfad_im_port_s *im_port, struct device *dev);
38void bfad_im_scsi_host_free(struct bfad_s *bfad, 38void bfad_im_scsi_host_free(struct bfad_s *bfad,
39 struct bfad_im_port_s *im_port); 39 struct bfad_im_port_s *im_port);
40 40
@@ -64,9 +64,11 @@ struct bfad_im_port_s {
64 struct work_struct port_delete_work; 64 struct work_struct port_delete_work;
65 int idr_id; 65 int idr_id;
66 u16 cur_scsi_id; 66 u16 cur_scsi_id;
67 u16 flags;
67 struct list_head binding_list; 68 struct list_head binding_list;
68 struct Scsi_Host *shost; 69 struct Scsi_Host *shost;
69 struct list_head itnim_mapped_list; 70 struct list_head itnim_mapped_list;
71 struct fc_vport *fc_vport;
70}; 72};
71 73
72enum bfad_itnim_state { 74enum bfad_itnim_state {
@@ -140,6 +142,8 @@ void bfad_im_itnim_unmap(struct bfad_im_port_s *im_port,
140extern struct scsi_host_template bfad_im_scsi_host_template; 142extern struct scsi_host_template bfad_im_scsi_host_template;
141extern struct scsi_host_template bfad_im_vport_template; 143extern struct scsi_host_template bfad_im_vport_template;
142extern struct fc_function_template bfad_im_fc_function_template; 144extern struct fc_function_template bfad_im_fc_function_template;
145extern struct fc_function_template bfad_im_vport_fc_function_template;
143extern struct scsi_transport_template *bfad_im_scsi_transport_template; 146extern struct scsi_transport_template *bfad_im_scsi_transport_template;
147extern struct scsi_transport_template *bfad_im_scsi_vport_transport_template;
144 148
145#endif 149#endif